%load_ext autoreload
%autoreload 2
#import necessary modules
import torch
import torch.nn as nn
from torchvision import transforms, datasets
from torch import optim as optim
# for visualization
from matplotlib import pyplot as plt
import math
import numpy as np
from torch.nn import functional as F
from sklearn.datasets import fetch_california_housing
from IPython.display import Image
from IPython.core.display import HTML
RESULTS_PATH = "/home/vs428/Documents/deeplearning_cpsc663/Assignment_3/results"
# import sample data
housing = fetch_california_housing()
m,n = housing.data.shape
housing_data_plus_bias = np.c_[np.ones((m,1)), housing.data]
X = torch.Tensor(housing['data'])
y = torch.Tensor(housing['target']).unsqueeze(1)
# create the weight vector
w_init = torch.randn(8,1,requires_grad=True)
# TO DO:
# a) calculate closed form gradient with respect to the weights
closed_w_grad = (2 * X.T @ X @ w_init) - (2 * X.T @ y)
print(closed_w_grad)
tensor([[-1.9356e+08],
[-1.2805e+09],
[-2.6385e+08],
[-5.3455e+07],
[-1.1534e+11],
[-1.8394e+08],
[-1.7675e+09],
[ 5.9536e+09]], grad_fn=<SubBackward0>)
y.shape
torch.Size([20640, 1])
# b) calculate gradient with respect to the weights w using autograd # first create the activation function
X_bar = X @ w_init
loss = ((y - X_bar)**2).sum()
loss.backward()
print(w_init.grad)
tensor([[-1.9356e+08],
[-1.2805e+09],
[-2.6385e+08],
[-5.3455e+07],
[-1.1534e+11],
[-1.8394e+08],
[-1.7675e+09],
[ 5.9536e+09]])
# c) check that the two are equal
torch.abs(closed_w_grad - w_init.grad) < 10000
tensor([[ True],
[ True],
[ True],
[ True],
[False],
[ True],
[ True],
[ True]])
Done in Tex file.
One simple example of a corruption process could be adding Gaussian noise with varying standard deviations. An equation for this is below:
\begin{equation}\label{eq:aenoise} C(\hat{x}\vert x) = x + \mathcal{N}(0,0.1) \end{equation}This function is implemented below on $y = x^2$
import numpy as np
def addGNoise(data, mu=0,sigma=0.1):
noise = np.random.default_rng().normal(mu, sigma, data.shape)
return data + noise
x = np.random.rand(100)
y = x**2
y_noisy = addGNoise(x**2)
fig, axs = plt.subplots(1,2, figsize=(10,5))
fig.suptitle("Adding Noise to Y=X^2", fontsize=16)
axs[0].scatter(x,y, c="blue")
axs[0].set_title("Y = X^2")
axs[1].scatter(x,y_noisy, c="red")
axs[1].set_title("Noisy Y=X^2")
plt.show()
class Autoencoder(nn.Module):
def __init__(self):
super(Autoencoder, self).__init__()
self.lin1 = nn.Linear(784, 2)
self.lin2 = nn.Linear(2,784)
def encode(self, x):
x = self.lin1(x)
return x
def decode(self, z):
z = torch.sigmoid(self.lin2(z))
return z
def forward(self, x):
z = self.encode(x)
return self.decode(z)
# tells PyTorch to use an NVIDIA GPU, if one is available.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device
device(type='cuda')
training_parameters = {
"img_size": 28,
"n_epochs": 24,
"batch_size": 64,
"learning_rate":1e-3
}
kwargs = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else {}
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('./data', train=True, download=True,
transform=transforms.ToTensor()),
batch_size=training_parameters['batch_size'], shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('./data', train=False, transform=transforms.ToTensor()),
batch_size=training_parameters['batch_size'], shuffle=True, **kwargs)
def train_autoencoder(model,loss_fn, optimizer, train_loader, test_loader):
"""
This is a standard training loop, which leaves some parts to be filled in.
INPUT:
:param model: an untrained pytorch model
:param loss_fn: e.g. Cross Entropy loss of Mean Squared Error.
:param optimizer: the model optimizer, initialized with a learning rate.
:param training_set: The training data, in a dataloader for easy iteration.
:param test_loader: The testing data, in a dataloader for easy iteration.
"""
num_epochs = 100 # obviously, this is too many. I don't know what this author was thinking.
for epoch in range(num_epochs):
# loop through each data point in the training set
for data, targets in train_loader:
optimizer.zero_grad()
# run the model on the data
data = torch.reshape(torch.squeeze(data), (-1,784))
model_input = data.to(device)
out = model(model_input)
# Calculate the loss
loss = loss_fn(out.cpu(),data)
# Find the gradients of our loss via backpropogation
loss.backward()
# Adjust accordingly with the optimizer
optimizer.step()
# Give status reports every 100 epochs
if epoch % 5==0:
print(f" EPOCH {epoch}. Progress: {epoch/num_epochs*100}%. ")
print(f" Reconstruction Loss: {loss.data}")
# Final model acc check
print(f" EPOCH {num_epochs}. Progress: 100%. ")
print(f" Reconstruction Loss: {loss.data}")
# initialize the model (adapt this to each model)
autoenc = Autoencoder().to(device)
# initialize the optimizer, and set the learning rate
adam_opt = torch.optim.Adam(autoenc.parameters(), lr=training_parameters['learning_rate'])
loss_fn = torch.nn.MSELoss()
train_autoencoder(autoenc, loss_fn, adam_opt, train_loader, test_loader)
EPOCH 0. Progress: 0.0%. Reconstruction Loss: 0.06711838394403458 EPOCH 5. Progress: 5.0%. Reconstruction Loss: 0.06170640513300896 EPOCH 10. Progress: 10.0%. Reconstruction Loss: 0.05072200670838356 EPOCH 15. Progress: 15.0%. Reconstruction Loss: 0.05124974250793457 EPOCH 20. Progress: 20.0%. Reconstruction Loss: 0.05451633408665657 EPOCH 25. Progress: 25.0%. Reconstruction Loss: 0.053853053599596024 EPOCH 30. Progress: 30.0%. Reconstruction Loss: 0.052599091082811356 EPOCH 35. Progress: 35.0%. Reconstruction Loss: 0.05498860776424408 EPOCH 40. Progress: 40.0%. Reconstruction Loss: 0.04880731552839279 EPOCH 45. Progress: 45.0%. Reconstruction Loss: 0.05510849878191948 EPOCH 50. Progress: 50.0%. Reconstruction Loss: 0.052627213299274445 EPOCH 55. Progress: 55.00000000000001%. Reconstruction Loss: 0.05164170265197754 EPOCH 60. Progress: 60.0%. Reconstruction Loss: 0.0596851222217083 EPOCH 65. Progress: 65.0%. Reconstruction Loss: 0.055819448083639145 EPOCH 70. Progress: 70.0%. Reconstruction Loss: 0.054233212023973465 EPOCH 75. Progress: 75.0%. Reconstruction Loss: 0.054976437240839005 EPOCH 80. Progress: 80.0%. Reconstruction Loss: 0.04565049707889557 EPOCH 85. Progress: 85.0%. Reconstruction Loss: 0.05807165056467056 EPOCH 90. Progress: 90.0%. Reconstruction Loss: 0.06087760627269745 EPOCH 95. Progress: 95.0%. Reconstruction Loss: 0.050877660512924194 EPOCH 100. Progress: 100%. Reconstruction Loss: 0.05272308737039566
embeddings = []
for data, targets in train_loader:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = data.to(device)
targets = targets.to(device)
model_input = data.view(data.shape[0], -1)
embeddings.append((autoenc.encode(model_input), targets))
i = 0
max = np.ceil(1500/128)
for gpu_data, gpu_labels in embeddings:
data = gpu_data.cpu().detach()
labels = gpu_labels.cpu().detach()
scatter = plt.scatter(data[:,0], data[:,1], c=labels)
i += 1
if i == max:
break
plt.legend(*scatter.legend_elements())
plt.show()
We can see that the AE learned some of the salient features of the dataset since we see a separation between the different digits, especially those that are very different from one another. For example, 0 and 1 are far apart, since they are very different. Same with 1 and 9, while 0 and 9 are closer since they are more similar. One of the issues we see here is that there isn't clear separation between the other points, so further optimization of the AE can be done. Furthermore, since we didn't use a VAE, we won't be able to get very reasonable examples if we sample from this latent space.
!python vae.py
!python vae.py --learning-rate 0.0005 --epochs 30 --batch-size 128
Train Epoch: 1 [0/60000 (0%)] Loss: 138.151978 Train Epoch: 1 [1280/60000 (2%)] Loss: 102.760048 Train Epoch: 1 [2560/60000 (4%)] Loss: 78.764740 Train Epoch: 1 [3840/60000 (6%)] Loss: 63.698444 Train Epoch: 1 [5120/60000 (9%)] Loss: 63.385769 Train Epoch: 1 [6400/60000 (11%)] Loss: 55.433586 Train Epoch: 1 [7680/60000 (13%)] Loss: 55.324188 Train Epoch: 1 [8960/60000 (15%)] Loss: 49.738503 Train Epoch: 1 [10240/60000 (17%)] Loss: 49.541695 Train Epoch: 1 [11520/60000 (19%)] Loss: 47.492535 Train Epoch: 1 [12800/60000 (21%)] Loss: 45.008034 Train Epoch: 1 [14080/60000 (23%)] Loss: 42.693081 Train Epoch: 1 [15360/60000 (26%)] Loss: 43.721050 Train Epoch: 1 [16640/60000 (28%)] Loss: 43.989265 Train Epoch: 1 [17920/60000 (30%)] Loss: 39.271080 Train Epoch: 1 [19200/60000 (32%)] Loss: 40.923557 Train Epoch: 1 [20480/60000 (34%)] Loss: 44.381531 Train Epoch: 1 [21760/60000 (36%)] Loss: 40.842110 Train Epoch: 1 [23040/60000 (38%)] Loss: 39.714394 Train Epoch: 1 [24320/60000 (41%)] Loss: 37.868259 Train Epoch: 1 [25600/60000 (43%)] Loss: 42.416851 Train Epoch: 1 [26880/60000 (45%)] Loss: 38.625374 Train Epoch: 1 [28160/60000 (47%)] Loss: 37.173241 Train Epoch: 1 [29440/60000 (49%)] Loss: 37.551071 Train Epoch: 1 [30720/60000 (51%)] Loss: 36.693348 Train Epoch: 1 [32000/60000 (53%)] Loss: 36.933434 Train Epoch: 1 [33280/60000 (55%)] Loss: 36.938042 Train Epoch: 1 [34560/60000 (58%)] Loss: 37.004402 Train Epoch: 1 [35840/60000 (60%)] Loss: 37.041611 Train Epoch: 1 [37120/60000 (62%)] Loss: 35.110523 Train Epoch: 1 [38400/60000 (64%)] Loss: 35.789997 Train Epoch: 1 [39680/60000 (66%)] Loss: 35.843056 Train Epoch: 1 [40960/60000 (68%)] Loss: 36.693882 Train Epoch: 1 [42240/60000 (70%)] Loss: 36.090412 Train Epoch: 1 [43520/60000 (72%)] Loss: 36.590858 Train Epoch: 1 [44800/60000 (75%)] Loss: 33.892220 Train Epoch: 1 [46080/60000 (77%)] Loss: 33.643658 Train Epoch: 1 [47360/60000 (79%)] Loss: 34.767769 Train Epoch: 1 [48640/60000 (81%)] Loss: 33.252682 Train Epoch: 1 [49920/60000 (83%)] Loss: 34.068859 Train Epoch: 1 [51200/60000 (85%)] Loss: 31.719551 Train Epoch: 1 [52480/60000 (87%)] Loss: 36.330479 Train Epoch: 1 [53760/60000 (90%)] Loss: 33.016571 Train Epoch: 1 [55040/60000 (92%)] Loss: 33.483864 Train Epoch: 1 [56320/60000 (94%)] Loss: 31.519806 Train Epoch: 1 [57600/60000 (96%)] Loss: 30.164785 Train Epoch: 1 [58880/60000 (98%)] Loss: 33.023670 ====> Epoch: 1 Average loss: 43.6912 ====> Test set loss: 33.5029 Train Epoch: 2 [0/60000 (0%)] Loss: 32.491589 Train Epoch: 2 [1280/60000 (2%)] Loss: 32.986435 Train Epoch: 2 [2560/60000 (4%)] Loss: 33.167061 Train Epoch: 2 [3840/60000 (6%)] Loss: 33.389217 Train Epoch: 2 [5120/60000 (9%)] Loss: 33.765957 Train Epoch: 2 [6400/60000 (11%)] Loss: 33.881416 Train Epoch: 2 [7680/60000 (13%)] Loss: 34.354736 Train Epoch: 2 [8960/60000 (15%)] Loss: 33.595295 Train Epoch: 2 [10240/60000 (17%)] Loss: 32.376839 Train Epoch: 2 [11520/60000 (19%)] Loss: 34.921223 Train Epoch: 2 [12800/60000 (21%)] Loss: 30.617558 Train Epoch: 2 [14080/60000 (23%)] Loss: 31.190470 Train Epoch: 2 [15360/60000 (26%)] Loss: 31.739304 Train Epoch: 2 [16640/60000 (28%)] Loss: 32.692131 Train Epoch: 2 [17920/60000 (30%)] Loss: 31.067410 Train Epoch: 2 [19200/60000 (32%)] Loss: 33.999855 Train Epoch: 2 [20480/60000 (34%)] Loss: 30.184368 Train Epoch: 2 [21760/60000 (36%)] Loss: 30.469770 Train Epoch: 2 [23040/60000 (38%)] Loss: 32.029160 Train Epoch: 2 [24320/60000 (41%)] Loss: 32.771278 Train Epoch: 2 [25600/60000 (43%)] Loss: 31.598509 Train Epoch: 2 [26880/60000 (45%)] Loss: 30.690353 Train Epoch: 2 [28160/60000 (47%)] Loss: 30.684725 Train Epoch: 2 [29440/60000 (49%)] Loss: 33.943504 Train Epoch: 2 [30720/60000 (51%)] Loss: 31.428484 Train Epoch: 2 [32000/60000 (53%)] Loss: 30.576340 Train Epoch: 2 [33280/60000 (55%)] Loss: 29.612024 Train Epoch: 2 [34560/60000 (58%)] Loss: 30.554523 Train Epoch: 2 [35840/60000 (60%)] Loss: 31.143248 Train Epoch: 2 [37120/60000 (62%)] Loss: 29.422523 Train Epoch: 2 [38400/60000 (64%)] Loss: 31.143223 Train Epoch: 2 [39680/60000 (66%)] Loss: 30.558233 Train Epoch: 2 [40960/60000 (68%)] Loss: 31.291370 Train Epoch: 2 [42240/60000 (70%)] Loss: 30.638887 Train Epoch: 2 [43520/60000 (72%)] Loss: 31.227573 Train Epoch: 2 [44800/60000 (75%)] Loss: 30.285435 Train Epoch: 2 [46080/60000 (77%)] Loss: 28.876717 Train Epoch: 2 [47360/60000 (79%)] Loss: 30.557917 Train Epoch: 2 [48640/60000 (81%)] Loss: 30.524212 Train Epoch: 2 [49920/60000 (83%)] Loss: 30.266041 Train Epoch: 2 [51200/60000 (85%)] Loss: 30.712246 Train Epoch: 2 [52480/60000 (87%)] Loss: 30.475122 Train Epoch: 2 [53760/60000 (90%)] Loss: 29.379284 Train Epoch: 2 [55040/60000 (92%)] Loss: 29.439928 Train Epoch: 2 [56320/60000 (94%)] Loss: 30.541321 Train Epoch: 2 [57600/60000 (96%)] Loss: 29.864914 Train Epoch: 2 [58880/60000 (98%)] Loss: 29.446285 ====> Epoch: 2 Average loss: 31.3733 ====> Test set loss: 29.9047 Train Epoch: 3 [0/60000 (0%)] Loss: 32.035080 Train Epoch: 3 [1280/60000 (2%)] Loss: 28.494785 Train Epoch: 3 [2560/60000 (4%)] Loss: 28.200705 Train Epoch: 3 [3840/60000 (6%)] Loss: 30.767818 Train Epoch: 3 [5120/60000 (9%)] Loss: 31.108391 Train Epoch: 3 [6400/60000 (11%)] Loss: 28.566402 Train Epoch: 3 [7680/60000 (13%)] Loss: 29.027365 Train Epoch: 3 [8960/60000 (15%)] Loss: 29.700121 Train Epoch: 3 [10240/60000 (17%)] Loss: 29.893703 Train Epoch: 3 [11520/60000 (19%)] Loss: 28.510818 Train Epoch: 3 [12800/60000 (21%)] Loss: 28.737083 Train Epoch: 3 [14080/60000 (23%)] Loss: 29.709883 Train Epoch: 3 [15360/60000 (26%)] Loss: 31.964617 Train Epoch: 3 [16640/60000 (28%)] Loss: 29.678486 Train Epoch: 3 [17920/60000 (30%)] Loss: 28.508245 Train Epoch: 3 [19200/60000 (32%)] Loss: 28.665268 Train Epoch: 3 [20480/60000 (34%)] Loss: 28.803501 Train Epoch: 3 [21760/60000 (36%)] Loss: 30.852350 Train Epoch: 3 [23040/60000 (38%)] Loss: 30.010664 Train Epoch: 3 [24320/60000 (41%)] Loss: 28.744246 Train Epoch: 3 [25600/60000 (43%)] Loss: 27.756229 Train Epoch: 3 [26880/60000 (45%)] Loss: 27.953598 Train Epoch: 3 [28160/60000 (47%)] Loss: 28.782396 Train Epoch: 3 [29440/60000 (49%)] Loss: 28.468391 Train Epoch: 3 [30720/60000 (51%)] Loss: 29.221956 Train Epoch: 3 [32000/60000 (53%)] Loss: 29.364994 Train Epoch: 3 [33280/60000 (55%)] Loss: 30.499828 Train Epoch: 3 [34560/60000 (58%)] Loss: 30.464958 Train Epoch: 3 [35840/60000 (60%)] Loss: 28.672413 Train Epoch: 3 [37120/60000 (62%)] Loss: 27.961458 Train Epoch: 3 [38400/60000 (64%)] Loss: 29.060045 Train Epoch: 3 [39680/60000 (66%)] Loss: 29.203133 Train Epoch: 3 [40960/60000 (68%)] Loss: 27.354685 Train Epoch: 3 [42240/60000 (70%)] Loss: 30.190910 Train Epoch: 3 [43520/60000 (72%)] Loss: 28.223648 Train Epoch: 3 [44800/60000 (75%)] Loss: 29.655327 Train Epoch: 3 [46080/60000 (77%)] Loss: 28.357670 Train Epoch: 3 [47360/60000 (79%)] Loss: 27.838736 Train Epoch: 3 [48640/60000 (81%)] Loss: 28.890385 Train Epoch: 3 [49920/60000 (83%)] Loss: 29.673330 Train Epoch: 3 [51200/60000 (85%)] Loss: 28.275608 Train Epoch: 3 [52480/60000 (87%)] Loss: 27.872425 Train Epoch: 3 [53760/60000 (90%)] Loss: 28.506950 Train Epoch: 3 [55040/60000 (92%)] Loss: 27.414257 Train Epoch: 3 [56320/60000 (94%)] Loss: 27.269432 Train Epoch: 3 [57600/60000 (96%)] Loss: 29.004692 Train Epoch: 3 [58880/60000 (98%)] Loss: 30.095364 ====> Epoch: 3 Average loss: 29.1382 ====> Test set loss: 28.6410 Train Epoch: 4 [0/60000 (0%)] Loss: 29.586964 Train Epoch: 4 [1280/60000 (2%)] Loss: 28.674629 Train Epoch: 4 [2560/60000 (4%)] Loss: 29.021667 Train Epoch: 4 [3840/60000 (6%)] Loss: 28.667408 Train Epoch: 4 [5120/60000 (9%)] Loss: 28.618782 Train Epoch: 4 [6400/60000 (11%)] Loss: 28.552197 Train Epoch: 4 [7680/60000 (13%)] Loss: 29.610296 Train Epoch: 4 [8960/60000 (15%)] Loss: 29.230371 Train Epoch: 4 [10240/60000 (17%)] Loss: 26.984015 Train Epoch: 4 [11520/60000 (19%)] Loss: 28.624462 Train Epoch: 4 [12800/60000 (21%)] Loss: 28.003220 Train Epoch: 4 [14080/60000 (23%)] Loss: 26.779823 Train Epoch: 4 [15360/60000 (26%)] Loss: 26.734104 Train Epoch: 4 [16640/60000 (28%)] Loss: 26.442047 Train Epoch: 4 [17920/60000 (30%)] Loss: 30.702326 Train Epoch: 4 [19200/60000 (32%)] Loss: 28.741629 Train Epoch: 4 [20480/60000 (34%)] Loss: 27.464418 Train Epoch: 4 [21760/60000 (36%)] Loss: 27.885429 Train Epoch: 4 [23040/60000 (38%)] Loss: 28.685946 Train Epoch: 4 [24320/60000 (41%)] Loss: 26.954071 Train Epoch: 4 [25600/60000 (43%)] Loss: 29.581890 Train Epoch: 4 [26880/60000 (45%)] Loss: 27.924124 Train Epoch: 4 [28160/60000 (47%)] Loss: 26.796875 Train Epoch: 4 [29440/60000 (49%)] Loss: 27.115206 Train Epoch: 4 [30720/60000 (51%)] Loss: 25.952049 Train Epoch: 4 [32000/60000 (53%)] Loss: 27.231396 Train Epoch: 4 [33280/60000 (55%)] Loss: 27.897091 Train Epoch: 4 [34560/60000 (58%)] Loss: 28.125641 Train Epoch: 4 [35840/60000 (60%)] Loss: 28.237610 Train Epoch: 4 [37120/60000 (62%)] Loss: 27.629402 Train Epoch: 4 [38400/60000 (64%)] Loss: 27.796524 Train Epoch: 4 [39680/60000 (66%)] Loss: 27.812159 Train Epoch: 4 [40960/60000 (68%)] Loss: 26.409698 Train Epoch: 4 [42240/60000 (70%)] Loss: 27.077599 Train Epoch: 4 [43520/60000 (72%)] Loss: 27.742168 Train Epoch: 4 [44800/60000 (75%)] Loss: 28.552040 Train Epoch: 4 [46080/60000 (77%)] Loss: 27.372070 Train Epoch: 4 [47360/60000 (79%)] Loss: 26.709080 Train Epoch: 4 [48640/60000 (81%)] Loss: 26.857031 Train Epoch: 4 [49920/60000 (83%)] Loss: 28.288223 Train Epoch: 4 [51200/60000 (85%)] Loss: 28.267164 Train Epoch: 4 [52480/60000 (87%)] Loss: 28.977596 Train Epoch: 4 [53760/60000 (90%)] Loss: 27.657013 Train Epoch: 4 [55040/60000 (92%)] Loss: 27.179256 Train Epoch: 4 [56320/60000 (94%)] Loss: 27.554518 Train Epoch: 4 [57600/60000 (96%)] Loss: 26.385132 Train Epoch: 4 [58880/60000 (98%)] Loss: 28.888329 ====> Epoch: 4 Average loss: 28.0380 ====> Test set loss: 27.6297 Train Epoch: 5 [0/60000 (0%)] Loss: 28.146461 Train Epoch: 5 [1280/60000 (2%)] Loss: 28.332945 Train Epoch: 5 [2560/60000 (4%)] Loss: 28.912285 Train Epoch: 5 [3840/60000 (6%)] Loss: 28.620590 Train Epoch: 5 [5120/60000 (9%)] Loss: 28.941809 Train Epoch: 5 [6400/60000 (11%)] Loss: 27.969429 Train Epoch: 5 [7680/60000 (13%)] Loss: 27.541441 Train Epoch: 5 [8960/60000 (15%)] Loss: 28.332506 Train Epoch: 5 [10240/60000 (17%)] Loss: 28.911221 Train Epoch: 5 [11520/60000 (19%)] Loss: 25.359631 Train Epoch: 5 [12800/60000 (21%)] Loss: 27.610521 Train Epoch: 5 [14080/60000 (23%)] Loss: 28.256819 Train Epoch: 5 [15360/60000 (26%)] Loss: 27.284033 Train Epoch: 5 [16640/60000 (28%)] Loss: 28.273619 Train Epoch: 5 [17920/60000 (30%)] Loss: 28.507376 Train Epoch: 5 [19200/60000 (32%)] Loss: 27.714848 Train Epoch: 5 [20480/60000 (34%)] Loss: 27.094833 Train Epoch: 5 [21760/60000 (36%)] Loss: 26.496157 Train Epoch: 5 [23040/60000 (38%)] Loss: 27.665323 Train Epoch: 5 [24320/60000 (41%)] Loss: 28.291416 Train Epoch: 5 [25600/60000 (43%)] Loss: 26.910393 Train Epoch: 5 [26880/60000 (45%)] Loss: 25.961433 Train Epoch: 5 [28160/60000 (47%)] Loss: 28.007395 Train Epoch: 5 [29440/60000 (49%)] Loss: 26.902519 Train Epoch: 5 [30720/60000 (51%)] Loss: 27.346096 Train Epoch: 5 [32000/60000 (53%)] Loss: 26.266453 Train Epoch: 5 [33280/60000 (55%)] Loss: 26.428734 Train Epoch: 5 [34560/60000 (58%)] Loss: 27.504330 Train Epoch: 5 [35840/60000 (60%)] Loss: 29.368717 Train Epoch: 5 [37120/60000 (62%)] Loss: 27.317587 Train Epoch: 5 [38400/60000 (64%)] Loss: 26.727791 Train Epoch: 5 [39680/60000 (66%)] Loss: 27.052788 Train Epoch: 5 [40960/60000 (68%)] Loss: 27.970819 Train Epoch: 5 [42240/60000 (70%)] Loss: 27.241795 Train Epoch: 5 [43520/60000 (72%)] Loss: 27.521629 Train Epoch: 5 [44800/60000 (75%)] Loss: 25.581959 Train Epoch: 5 [46080/60000 (77%)] Loss: 26.244984 Train Epoch: 5 [47360/60000 (79%)] Loss: 27.711826 Train Epoch: 5 [48640/60000 (81%)] Loss: 28.712183 Train Epoch: 5 [49920/60000 (83%)] Loss: 27.008081 Train Epoch: 5 [51200/60000 (85%)] Loss: 27.570515 Train Epoch: 5 [52480/60000 (87%)] Loss: 27.555084 Train Epoch: 5 [53760/60000 (90%)] Loss: 26.289017 Train Epoch: 5 [55040/60000 (92%)] Loss: 26.099277 Train Epoch: 5 [56320/60000 (94%)] Loss: 26.798225 Train Epoch: 5 [57600/60000 (96%)] Loss: 27.390606 Train Epoch: 5 [58880/60000 (98%)] Loss: 24.854494 ====> Epoch: 5 Average loss: 27.4154 ====> Test set loss: 27.2176 Train Epoch: 6 [0/60000 (0%)] Loss: 27.507133 Train Epoch: 6 [1280/60000 (2%)] Loss: 26.073280 Train Epoch: 6 [2560/60000 (4%)] Loss: 27.041813 Train Epoch: 6 [3840/60000 (6%)] Loss: 27.699936 Train Epoch: 6 [5120/60000 (9%)] Loss: 28.402582 Train Epoch: 6 [6400/60000 (11%)] Loss: 27.538151 Train Epoch: 6 [7680/60000 (13%)] Loss: 27.262447 Train Epoch: 6 [8960/60000 (15%)] Loss: 27.171656 Train Epoch: 6 [10240/60000 (17%)] Loss: 27.150145 Train Epoch: 6 [11520/60000 (19%)] Loss: 27.216440 Train Epoch: 6 [12800/60000 (21%)] Loss: 26.710459 Train Epoch: 6 [14080/60000 (23%)] Loss: 26.874168 Train Epoch: 6 [15360/60000 (26%)] Loss: 26.810787 Train Epoch: 6 [16640/60000 (28%)] Loss: 25.811638 Train Epoch: 6 [17920/60000 (30%)] Loss: 28.309946 Train Epoch: 6 [19200/60000 (32%)] Loss: 28.074963 Train Epoch: 6 [20480/60000 (34%)] Loss: 25.775206 Train Epoch: 6 [21760/60000 (36%)] Loss: 26.531868 Train Epoch: 6 [23040/60000 (38%)] Loss: 26.611387 Train Epoch: 6 [24320/60000 (41%)] Loss: 26.491396 Train Epoch: 6 [25600/60000 (43%)] Loss: 28.337757 Train Epoch: 6 [26880/60000 (45%)] Loss: 27.228327 Train Epoch: 6 [28160/60000 (47%)] Loss: 27.210735 Train Epoch: 6 [29440/60000 (49%)] Loss: 26.770508 Train Epoch: 6 [30720/60000 (51%)] Loss: 27.988224 Train Epoch: 6 [32000/60000 (53%)] Loss: 28.790514 Train Epoch: 6 [33280/60000 (55%)] Loss: 27.757492 Train Epoch: 6 [34560/60000 (58%)] Loss: 27.625362 Train Epoch: 6 [35840/60000 (60%)] Loss: 26.759626 Train Epoch: 6 [37120/60000 (62%)] Loss: 25.653900 Train Epoch: 6 [38400/60000 (64%)] Loss: 26.810753 Train Epoch: 6 [39680/60000 (66%)] Loss: 27.291908 Train Epoch: 6 [40960/60000 (68%)] Loss: 26.062889 Train Epoch: 6 [42240/60000 (70%)] Loss: 25.514746 Train Epoch: 6 [43520/60000 (72%)] Loss: 26.322081 Train Epoch: 6 [44800/60000 (75%)] Loss: 27.460133 Train Epoch: 6 [46080/60000 (77%)] Loss: 27.787430 Train Epoch: 6 [47360/60000 (79%)] Loss: 27.487492 Train Epoch: 6 [48640/60000 (81%)] Loss: 26.233673 Train Epoch: 6 [49920/60000 (83%)] Loss: 26.792229 Train Epoch: 6 [51200/60000 (85%)] Loss: 24.727821 Train Epoch: 6 [52480/60000 (87%)] Loss: 26.785828 Train Epoch: 6 [53760/60000 (90%)] Loss: 27.034897 Train Epoch: 6 [55040/60000 (92%)] Loss: 25.677895 Train Epoch: 6 [56320/60000 (94%)] Loss: 28.065403 Train Epoch: 6 [57600/60000 (96%)] Loss: 25.888258 Train Epoch: 6 [58880/60000 (98%)] Loss: 26.422749 ====> Epoch: 6 Average loss: 26.9850 ====> Test set loss: 26.7398 Train Epoch: 7 [0/60000 (0%)] Loss: 26.860476 Train Epoch: 7 [1280/60000 (2%)] Loss: 27.413389 Train Epoch: 7 [2560/60000 (4%)] Loss: 26.996284 Train Epoch: 7 [3840/60000 (6%)] Loss: 26.190306 Train Epoch: 7 [5120/60000 (9%)] Loss: 28.033548 Train Epoch: 7 [6400/60000 (11%)] Loss: 28.611256 Train Epoch: 7 [7680/60000 (13%)] Loss: 26.233273 Train Epoch: 7 [8960/60000 (15%)] Loss: 27.468090 Train Epoch: 7 [10240/60000 (17%)] Loss: 27.203396 Train Epoch: 7 [11520/60000 (19%)] Loss: 27.164389 Train Epoch: 7 [12800/60000 (21%)] Loss: 28.044725 Train Epoch: 7 [14080/60000 (23%)] Loss: 26.089840 Train Epoch: 7 [15360/60000 (26%)] Loss: 26.537598 Train Epoch: 7 [16640/60000 (28%)] Loss: 25.831448 Train Epoch: 7 [17920/60000 (30%)] Loss: 27.168795 Train Epoch: 7 [19200/60000 (32%)] Loss: 26.365971 Train Epoch: 7 [20480/60000 (34%)] Loss: 26.923597 Train Epoch: 7 [21760/60000 (36%)] Loss: 26.822109 Train Epoch: 7 [23040/60000 (38%)] Loss: 26.049545 Train Epoch: 7 [24320/60000 (41%)] Loss: 24.494823 Train Epoch: 7 [25600/60000 (43%)] Loss: 26.228041 Train Epoch: 7 [26880/60000 (45%)] Loss: 25.699903 Train Epoch: 7 [28160/60000 (47%)] Loss: 28.063007 Train Epoch: 7 [29440/60000 (49%)] Loss: 27.542650 Train Epoch: 7 [30720/60000 (51%)] Loss: 27.164312 Train Epoch: 7 [32000/60000 (53%)] Loss: 27.008801 Train Epoch: 7 [33280/60000 (55%)] Loss: 25.708693 Train Epoch: 7 [34560/60000 (58%)] Loss: 25.907116 Train Epoch: 7 [35840/60000 (60%)] Loss: 27.036652 Train Epoch: 7 [37120/60000 (62%)] Loss: 26.079102 Train Epoch: 7 [38400/60000 (64%)] Loss: 27.338146 Train Epoch: 7 [39680/60000 (66%)] Loss: 27.859015 Train Epoch: 7 [40960/60000 (68%)] Loss: 28.074924 Train Epoch: 7 [42240/60000 (70%)] Loss: 26.573025 Train Epoch: 7 [43520/60000 (72%)] Loss: 25.242708 Train Epoch: 7 [44800/60000 (75%)] Loss: 27.219515 Train Epoch: 7 [46080/60000 (77%)] Loss: 27.405552 Train Epoch: 7 [47360/60000 (79%)] Loss: 26.118269 Train Epoch: 7 [48640/60000 (81%)] Loss: 26.760544 Train Epoch: 7 [49920/60000 (83%)] Loss: 25.685551 Train Epoch: 7 [51200/60000 (85%)] Loss: 26.218544 Train Epoch: 7 [52480/60000 (87%)] Loss: 27.379051 Train Epoch: 7 [53760/60000 (90%)] Loss: 24.655712 Train Epoch: 7 [55040/60000 (92%)] Loss: 26.765381 Train Epoch: 7 [56320/60000 (94%)] Loss: 26.627094 Train Epoch: 7 [57600/60000 (96%)] Loss: 28.272614 Train Epoch: 7 [58880/60000 (98%)] Loss: 27.539021 ====> Epoch: 7 Average loss: 26.6596 ====> Test set loss: 26.5249 Train Epoch: 8 [0/60000 (0%)] Loss: 27.655607 Train Epoch: 8 [1280/60000 (2%)] Loss: 25.873264 Train Epoch: 8 [2560/60000 (4%)] Loss: 26.627451 Train Epoch: 8 [3840/60000 (6%)] Loss: 28.277321 Train Epoch: 8 [5120/60000 (9%)] Loss: 25.541405 Train Epoch: 8 [6400/60000 (11%)] Loss: 26.417883 Train Epoch: 8 [7680/60000 (13%)] Loss: 27.971060 Train Epoch: 8 [8960/60000 (15%)] Loss: 25.335369 Train Epoch: 8 [10240/60000 (17%)] Loss: 25.815044 Train Epoch: 8 [11520/60000 (19%)] Loss: 26.809233 Train Epoch: 8 [12800/60000 (21%)] Loss: 27.227356 Train Epoch: 8 [14080/60000 (23%)] Loss: 26.355040 Train Epoch: 8 [15360/60000 (26%)] Loss: 26.594898 Train Epoch: 8 [16640/60000 (28%)] Loss: 26.728893 Train Epoch: 8 [17920/60000 (30%)] Loss: 26.880413 Train Epoch: 8 [19200/60000 (32%)] Loss: 26.098289 Train Epoch: 8 [20480/60000 (34%)] Loss: 26.155373 Train Epoch: 8 [21760/60000 (36%)] Loss: 25.963860 Train Epoch: 8 [23040/60000 (38%)] Loss: 26.077002 Train Epoch: 8 [24320/60000 (41%)] Loss: 25.969343 Train Epoch: 8 [25600/60000 (43%)] Loss: 25.990313 Train Epoch: 8 [26880/60000 (45%)] Loss: 25.807859 Train Epoch: 8 [28160/60000 (47%)] Loss: 25.395914 Train Epoch: 8 [29440/60000 (49%)] Loss: 26.098057 Train Epoch: 8 [30720/60000 (51%)] Loss: 24.641842 Train Epoch: 8 [32000/60000 (53%)] Loss: 26.405514 Train Epoch: 8 [33280/60000 (55%)] Loss: 27.378845 Train Epoch: 8 [34560/60000 (58%)] Loss: 25.254929 Train Epoch: 8 [35840/60000 (60%)] Loss: 26.240124 Train Epoch: 8 [37120/60000 (62%)] Loss: 26.450459 Train Epoch: 8 [38400/60000 (64%)] Loss: 27.185099 Train Epoch: 8 [39680/60000 (66%)] Loss: 28.043381 Train Epoch: 8 [40960/60000 (68%)] Loss: 27.117954 Train Epoch: 8 [42240/60000 (70%)] Loss: 26.699945 Train Epoch: 8 [43520/60000 (72%)] Loss: 26.438377 Train Epoch: 8 [44800/60000 (75%)] Loss: 26.625216 Train Epoch: 8 [46080/60000 (77%)] Loss: 26.332642 Train Epoch: 8 [47360/60000 (79%)] Loss: 27.216291 Train Epoch: 8 [48640/60000 (81%)] Loss: 26.308512 Train Epoch: 8 [49920/60000 (83%)] Loss: 26.965076 Train Epoch: 8 [51200/60000 (85%)] Loss: 26.060081 Train Epoch: 8 [52480/60000 (87%)] Loss: 27.175417 Train Epoch: 8 [53760/60000 (90%)] Loss: 25.409515 Train Epoch: 8 [55040/60000 (92%)] Loss: 26.733534 Train Epoch: 8 [56320/60000 (94%)] Loss: 27.005297 Train Epoch: 8 [57600/60000 (96%)] Loss: 27.426573 Train Epoch: 8 [58880/60000 (98%)] Loss: 25.831615 ====> Epoch: 8 Average loss: 26.4488 ====> Test set loss: 26.5123 Train Epoch: 9 [0/60000 (0%)] Loss: 26.864769 Train Epoch: 9 [1280/60000 (2%)] Loss: 25.750093 Train Epoch: 9 [2560/60000 (4%)] Loss: 25.448484 Train Epoch: 9 [3840/60000 (6%)] Loss: 25.141701 Train Epoch: 9 [5120/60000 (9%)] Loss: 25.931919 Train Epoch: 9 [6400/60000 (11%)] Loss: 26.100254 Train Epoch: 9 [7680/60000 (13%)] Loss: 26.218679 Train Epoch: 9 [8960/60000 (15%)] Loss: 26.259232 Train Epoch: 9 [10240/60000 (17%)] Loss: 25.442078 Train Epoch: 9 [11520/60000 (19%)] Loss: 25.266529 Train Epoch: 9 [12800/60000 (21%)] Loss: 26.803743 Train Epoch: 9 [14080/60000 (23%)] Loss: 26.009462 Train Epoch: 9 [15360/60000 (26%)] Loss: 26.577490 Train Epoch: 9 [16640/60000 (28%)] Loss: 27.333969 Train Epoch: 9 [17920/60000 (30%)] Loss: 26.695202 Train Epoch: 9 [19200/60000 (32%)] Loss: 25.780481 Train Epoch: 9 [20480/60000 (34%)] Loss: 27.672451 Train Epoch: 9 [21760/60000 (36%)] Loss: 25.260998 Train Epoch: 9 [23040/60000 (38%)] Loss: 24.361786 Train Epoch: 9 [24320/60000 (41%)] Loss: 26.353939 Train Epoch: 9 [25600/60000 (43%)] Loss: 25.009716 Train Epoch: 9 [26880/60000 (45%)] Loss: 26.081690 Train Epoch: 9 [28160/60000 (47%)] Loss: 27.639267 Train Epoch: 9 [29440/60000 (49%)] Loss: 25.995329 Train Epoch: 9 [30720/60000 (51%)] Loss: 26.364470 Train Epoch: 9 [32000/60000 (53%)] Loss: 26.339113 Train Epoch: 9 [33280/60000 (55%)] Loss: 26.218403 Train Epoch: 9 [34560/60000 (58%)] Loss: 25.382492 Train Epoch: 9 [35840/60000 (60%)] Loss: 25.980831 Train Epoch: 9 [37120/60000 (62%)] Loss: 25.607889 Train Epoch: 9 [38400/60000 (64%)] Loss: 26.867270 Train Epoch: 9 [39680/60000 (66%)] Loss: 25.867865 Train Epoch: 9 [40960/60000 (68%)] Loss: 25.873058 Train Epoch: 9 [42240/60000 (70%)] Loss: 25.838247 Train Epoch: 9 [43520/60000 (72%)] Loss: 25.220238 Train Epoch: 9 [44800/60000 (75%)] Loss: 25.465881 Train Epoch: 9 [46080/60000 (77%)] Loss: 25.594566 Train Epoch: 9 [47360/60000 (79%)] Loss: 26.328285 Train Epoch: 9 [48640/60000 (81%)] Loss: 26.581997 Train Epoch: 9 [49920/60000 (83%)] Loss: 24.950691 Train Epoch: 9 [51200/60000 (85%)] Loss: 25.348421 Train Epoch: 9 [52480/60000 (87%)] Loss: 26.374903 Train Epoch: 9 [53760/60000 (90%)] Loss: 25.767200 Train Epoch: 9 [55040/60000 (92%)] Loss: 26.532129 Train Epoch: 9 [56320/60000 (94%)] Loss: 25.937975 Train Epoch: 9 [57600/60000 (96%)] Loss: 25.298170 Train Epoch: 9 [58880/60000 (98%)] Loss: 25.879480 ====> Epoch: 9 Average loss: 26.1962 ====> Test set loss: 26.2485 Train Epoch: 10 [0/60000 (0%)] Loss: 24.919737 Train Epoch: 10 [1280/60000 (2%)] Loss: 25.884626 Train Epoch: 10 [2560/60000 (4%)] Loss: 27.048466 Train Epoch: 10 [3840/60000 (6%)] Loss: 25.375814 Train Epoch: 10 [5120/60000 (9%)] Loss: 27.744076 Train Epoch: 10 [6400/60000 (11%)] Loss: 27.988480 Train Epoch: 10 [7680/60000 (13%)] Loss: 26.860126 Train Epoch: 10 [8960/60000 (15%)] Loss: 25.719641 Train Epoch: 10 [10240/60000 (17%)] Loss: 24.540424 Train Epoch: 10 [11520/60000 (19%)] Loss: 25.319126 Train Epoch: 10 [12800/60000 (21%)] Loss: 26.156591 Train Epoch: 10 [14080/60000 (23%)] Loss: 25.082743 Train Epoch: 10 [15360/60000 (26%)] Loss: 25.711485 Train Epoch: 10 [16640/60000 (28%)] Loss: 27.284624 Train Epoch: 10 [17920/60000 (30%)] Loss: 26.769253 Train Epoch: 10 [19200/60000 (32%)] Loss: 26.858715 Train Epoch: 10 [20480/60000 (34%)] Loss: 26.996670 Train Epoch: 10 [21760/60000 (36%)] Loss: 24.814526 Train Epoch: 10 [23040/60000 (38%)] Loss: 24.134562 Train Epoch: 10 [24320/60000 (41%)] Loss: 24.872490 Train Epoch: 10 [25600/60000 (43%)] Loss: 26.433285 Train Epoch: 10 [26880/60000 (45%)] Loss: 25.161564 Train Epoch: 10 [28160/60000 (47%)] Loss: 26.592291 Train Epoch: 10 [29440/60000 (49%)] Loss: 25.859854 Train Epoch: 10 [30720/60000 (51%)] Loss: 26.134743 Train Epoch: 10 [32000/60000 (53%)] Loss: 25.870743 Train Epoch: 10 [33280/60000 (55%)] Loss: 26.145142 Train Epoch: 10 [34560/60000 (58%)] Loss: 27.694996 Train Epoch: 10 [35840/60000 (60%)] Loss: 27.402901 Train Epoch: 10 [37120/60000 (62%)] Loss: 26.756683 Train Epoch: 10 [38400/60000 (64%)] Loss: 25.719685 Train Epoch: 10 [39680/60000 (66%)] Loss: 25.539452 Train Epoch: 10 [40960/60000 (68%)] Loss: 25.614464 Train Epoch: 10 [42240/60000 (70%)] Loss: 24.906572 Train Epoch: 10 [43520/60000 (72%)] Loss: 25.840408 Train Epoch: 10 [44800/60000 (75%)] Loss: 26.414715 Train Epoch: 10 [46080/60000 (77%)] Loss: 26.351231 Train Epoch: 10 [47360/60000 (79%)] Loss: 26.124931 Train Epoch: 10 [48640/60000 (81%)] Loss: 26.971880 Train Epoch: 10 [49920/60000 (83%)] Loss: 26.078300 Train Epoch: 10 [51200/60000 (85%)] Loss: 24.176758 Train Epoch: 10 [52480/60000 (87%)] Loss: 27.048046 Train Epoch: 10 [53760/60000 (90%)] Loss: 26.879095 Train Epoch: 10 [55040/60000 (92%)] Loss: 24.894194 Train Epoch: 10 [56320/60000 (94%)] Loss: 26.169163 Train Epoch: 10 [57600/60000 (96%)] Loss: 26.313292 Train Epoch: 10 [58880/60000 (98%)] Loss: 25.704971 ====> Epoch: 10 Average loss: 26.0063 ====> Test set loss: 25.9012 Train Epoch: 11 [0/60000 (0%)] Loss: 25.797676 Train Epoch: 11 [1280/60000 (2%)] Loss: 26.880836 Train Epoch: 11 [2560/60000 (4%)] Loss: 25.456404 Train Epoch: 11 [3840/60000 (6%)] Loss: 27.451052 Train Epoch: 11 [5120/60000 (9%)] Loss: 26.251015 Train Epoch: 11 [6400/60000 (11%)] Loss: 25.564413 Train Epoch: 11 [7680/60000 (13%)] Loss: 25.489479 Train Epoch: 11 [8960/60000 (15%)] Loss: 26.599672 Train Epoch: 11 [10240/60000 (17%)] Loss: 25.378929 Train Epoch: 11 [11520/60000 (19%)] Loss: 24.890285 Train Epoch: 11 [12800/60000 (21%)] Loss: 26.977932 Train Epoch: 11 [14080/60000 (23%)] Loss: 25.777155 Train Epoch: 11 [15360/60000 (26%)] Loss: 25.452160 Train Epoch: 11 [16640/60000 (28%)] Loss: 25.457130 Train Epoch: 11 [17920/60000 (30%)] Loss: 27.585117 Train Epoch: 11 [19200/60000 (32%)] Loss: 26.597927 Train Epoch: 11 [20480/60000 (34%)] Loss: 26.254208 Train Epoch: 11 [21760/60000 (36%)] Loss: 24.643333 Train Epoch: 11 [23040/60000 (38%)] Loss: 25.779484 Train Epoch: 11 [24320/60000 (41%)] Loss: 24.956181 Train Epoch: 11 [25600/60000 (43%)] Loss: 25.462402 Train Epoch: 11 [26880/60000 (45%)] Loss: 26.152374 Train Epoch: 11 [28160/60000 (47%)] Loss: 25.189625 Train Epoch: 11 [29440/60000 (49%)] Loss: 25.309875 Train Epoch: 11 [30720/60000 (51%)] Loss: 25.330778 Train Epoch: 11 [32000/60000 (53%)] Loss: 26.371061 Train Epoch: 11 [33280/60000 (55%)] Loss: 24.741533 Train Epoch: 11 [34560/60000 (58%)] Loss: 26.051134 Train Epoch: 11 [35840/60000 (60%)] Loss: 25.411522 Train Epoch: 11 [37120/60000 (62%)] Loss: 23.727961 Train Epoch: 11 [38400/60000 (64%)] Loss: 24.924763 Train Epoch: 11 [39680/60000 (66%)] Loss: 26.769520 Train Epoch: 11 [40960/60000 (68%)] Loss: 26.066799 Train Epoch: 11 [42240/60000 (70%)] Loss: 27.920256 Train Epoch: 11 [43520/60000 (72%)] Loss: 25.163044 Train Epoch: 11 [44800/60000 (75%)] Loss: 26.373133 Train Epoch: 11 [46080/60000 (77%)] Loss: 25.153343 Train Epoch: 11 [47360/60000 (79%)] Loss: 26.543598 Train Epoch: 11 [48640/60000 (81%)] Loss: 24.401875 Train Epoch: 11 [49920/60000 (83%)] Loss: 25.272163 Train Epoch: 11 [51200/60000 (85%)] Loss: 25.968706 Train Epoch: 11 [52480/60000 (87%)] Loss: 25.910919 Train Epoch: 11 [53760/60000 (90%)] Loss: 27.764715 Train Epoch: 11 [55040/60000 (92%)] Loss: 26.247496 Train Epoch: 11 [56320/60000 (94%)] Loss: 26.994110 Train Epoch: 11 [57600/60000 (96%)] Loss: 26.328587 Train Epoch: 11 [58880/60000 (98%)] Loss: 25.271877 ====> Epoch: 11 Average loss: 25.8634 ====> Test set loss: 25.9646 Train Epoch: 12 [0/60000 (0%)] Loss: 25.947144 Train Epoch: 12 [1280/60000 (2%)] Loss: 25.849037 Train Epoch: 12 [2560/60000 (4%)] Loss: 24.805798 Train Epoch: 12 [3840/60000 (6%)] Loss: 25.881454 Train Epoch: 12 [5120/60000 (9%)] Loss: 23.692532 Train Epoch: 12 [6400/60000 (11%)] Loss: 25.756205 Train Epoch: 12 [7680/60000 (13%)] Loss: 26.315163 Train Epoch: 12 [8960/60000 (15%)] Loss: 26.982519 Train Epoch: 12 [10240/60000 (17%)] Loss: 26.120724 Train Epoch: 12 [11520/60000 (19%)] Loss: 26.346188 Train Epoch: 12 [12800/60000 (21%)] Loss: 26.514795 Train Epoch: 12 [14080/60000 (23%)] Loss: 25.963358 Train Epoch: 12 [15360/60000 (26%)] Loss: 25.727041 Train Epoch: 12 [16640/60000 (28%)] Loss: 25.271124 Train Epoch: 12 [17920/60000 (30%)] Loss: 25.602709 Train Epoch: 12 [19200/60000 (32%)] Loss: 26.115208 Train Epoch: 12 [20480/60000 (34%)] Loss: 24.726110 Train Epoch: 12 [21760/60000 (36%)] Loss: 24.761703 Train Epoch: 12 [23040/60000 (38%)] Loss: 24.206821 Train Epoch: 12 [24320/60000 (41%)] Loss: 26.473904 Train Epoch: 12 [25600/60000 (43%)] Loss: 25.150578 Train Epoch: 12 [26880/60000 (45%)] Loss: 25.197731 Train Epoch: 12 [28160/60000 (47%)] Loss: 25.741741 Train Epoch: 12 [29440/60000 (49%)] Loss: 25.968122 Train Epoch: 12 [30720/60000 (51%)] Loss: 24.899982 Train Epoch: 12 [32000/60000 (53%)] Loss: 26.042313 Train Epoch: 12 [33280/60000 (55%)] Loss: 25.999004 Train Epoch: 12 [34560/60000 (58%)] Loss: 26.396862 Train Epoch: 12 [35840/60000 (60%)] Loss: 25.332476 Train Epoch: 12 [37120/60000 (62%)] Loss: 25.062475 Train Epoch: 12 [38400/60000 (64%)] Loss: 26.563057 Train Epoch: 12 [39680/60000 (66%)] Loss: 25.128759 Train Epoch: 12 [40960/60000 (68%)] Loss: 25.860352 Train Epoch: 12 [42240/60000 (70%)] Loss: 26.002230 Train Epoch: 12 [43520/60000 (72%)] Loss: 25.944923 Train Epoch: 12 [44800/60000 (75%)] Loss: 25.466671 Train Epoch: 12 [46080/60000 (77%)] Loss: 25.397945 Train Epoch: 12 [47360/60000 (79%)] Loss: 26.094753 Train Epoch: 12 [48640/60000 (81%)] Loss: 26.881451 Train Epoch: 12 [49920/60000 (83%)] Loss: 26.029558 Train Epoch: 12 [51200/60000 (85%)] Loss: 25.513363 Train Epoch: 12 [52480/60000 (87%)] Loss: 25.264280 Train Epoch: 12 [53760/60000 (90%)] Loss: 26.036434 Train Epoch: 12 [55040/60000 (92%)] Loss: 23.888428 Train Epoch: 12 [56320/60000 (94%)] Loss: 24.170155 Train Epoch: 12 [57600/60000 (96%)] Loss: 27.195175 Train Epoch: 12 [58880/60000 (98%)] Loss: 25.685699 ====> Epoch: 12 Average loss: 25.6927 ====> Test set loss: 25.6335 Train Epoch: 13 [0/60000 (0%)] Loss: 24.789837 Train Epoch: 13 [1280/60000 (2%)] Loss: 24.693441 Train Epoch: 13 [2560/60000 (4%)] Loss: 25.569496 Train Epoch: 13 [3840/60000 (6%)] Loss: 24.143888 Train Epoch: 13 [5120/60000 (9%)] Loss: 25.153492 Train Epoch: 13 [6400/60000 (11%)] Loss: 25.334621 Train Epoch: 13 [7680/60000 (13%)] Loss: 24.881416 Train Epoch: 13 [8960/60000 (15%)] Loss: 25.915316 Train Epoch: 13 [10240/60000 (17%)] Loss: 24.354088 Train Epoch: 13 [11520/60000 (19%)] Loss: 26.545357 Train Epoch: 13 [12800/60000 (21%)] Loss: 25.986853 Train Epoch: 13 [14080/60000 (23%)] Loss: 25.902029 Train Epoch: 13 [15360/60000 (26%)] Loss: 26.563377 Train Epoch: 13 [16640/60000 (28%)] Loss: 25.782356 Train Epoch: 13 [17920/60000 (30%)] Loss: 26.734028 Train Epoch: 13 [19200/60000 (32%)] Loss: 25.689142 Train Epoch: 13 [20480/60000 (34%)] Loss: 26.237911 Train Epoch: 13 [21760/60000 (36%)] Loss: 26.857189 Train Epoch: 13 [23040/60000 (38%)] Loss: 25.772499 Train Epoch: 13 [24320/60000 (41%)] Loss: 25.591536 Train Epoch: 13 [25600/60000 (43%)] Loss: 24.280445 Train Epoch: 13 [26880/60000 (45%)] Loss: 27.111235 Train Epoch: 13 [28160/60000 (47%)] Loss: 25.207027 Train Epoch: 13 [29440/60000 (49%)] Loss: 25.651539 Train Epoch: 13 [30720/60000 (51%)] Loss: 24.337957 Train Epoch: 13 [32000/60000 (53%)] Loss: 24.455189 Train Epoch: 13 [33280/60000 (55%)] Loss: 27.588741 Train Epoch: 13 [34560/60000 (58%)] Loss: 26.451870 Train Epoch: 13 [35840/60000 (60%)] Loss: 25.358690 Train Epoch: 13 [37120/60000 (62%)] Loss: 25.690073 Train Epoch: 13 [38400/60000 (64%)] Loss: 24.106522 Train Epoch: 13 [39680/60000 (66%)] Loss: 25.104862 Train Epoch: 13 [40960/60000 (68%)] Loss: 26.373837 Train Epoch: 13 [42240/60000 (70%)] Loss: 27.342361 Train Epoch: 13 [43520/60000 (72%)] Loss: 24.364765 Train Epoch: 13 [44800/60000 (75%)] Loss: 25.129061 Train Epoch: 13 [46080/60000 (77%)] Loss: 26.092514 Train Epoch: 13 [47360/60000 (79%)] Loss: 27.134672 Train Epoch: 13 [48640/60000 (81%)] Loss: 25.361656 Train Epoch: 13 [49920/60000 (83%)] Loss: 26.281370 Train Epoch: 13 [51200/60000 (85%)] Loss: 25.431938 Train Epoch: 13 [52480/60000 (87%)] Loss: 25.690342 Train Epoch: 13 [53760/60000 (90%)] Loss: 25.431540 Train Epoch: 13 [55040/60000 (92%)] Loss: 26.116177 Train Epoch: 13 [56320/60000 (94%)] Loss: 24.738819 Train Epoch: 13 [57600/60000 (96%)] Loss: 24.876652 Train Epoch: 13 [58880/60000 (98%)] Loss: 25.226309 ====> Epoch: 13 Average loss: 25.6020 ====> Test set loss: 25.5706 Train Epoch: 14 [0/60000 (0%)] Loss: 24.867714 Train Epoch: 14 [1280/60000 (2%)] Loss: 24.473316 Train Epoch: 14 [2560/60000 (4%)] Loss: 26.887962 Train Epoch: 14 [3840/60000 (6%)] Loss: 25.451260 Train Epoch: 14 [5120/60000 (9%)] Loss: 24.648046 Train Epoch: 14 [6400/60000 (11%)] Loss: 26.419262 Train Epoch: 14 [7680/60000 (13%)] Loss: 25.754543 Train Epoch: 14 [8960/60000 (15%)] Loss: 24.466454 Train Epoch: 14 [10240/60000 (17%)] Loss: 26.599836 Train Epoch: 14 [11520/60000 (19%)] Loss: 25.129684 Train Epoch: 14 [12800/60000 (21%)] Loss: 25.972885 Train Epoch: 14 [14080/60000 (23%)] Loss: 25.408272 Train Epoch: 14 [15360/60000 (26%)] Loss: 25.567268 Train Epoch: 14 [16640/60000 (28%)] Loss: 25.585506 Train Epoch: 14 [17920/60000 (30%)] Loss: 24.420111 Train Epoch: 14 [19200/60000 (32%)] Loss: 24.958235 Train Epoch: 14 [20480/60000 (34%)] Loss: 25.725235 Train Epoch: 14 [21760/60000 (36%)] Loss: 25.808294 Train Epoch: 14 [23040/60000 (38%)] Loss: 25.357647 Train Epoch: 14 [24320/60000 (41%)] Loss: 25.147392 Train Epoch: 14 [25600/60000 (43%)] Loss: 26.035587 Train Epoch: 14 [26880/60000 (45%)] Loss: 24.108181 Train Epoch: 14 [28160/60000 (47%)] Loss: 24.874203 Train Epoch: 14 [29440/60000 (49%)] Loss: 25.381212 Train Epoch: 14 [30720/60000 (51%)] Loss: 24.601562 Train Epoch: 14 [32000/60000 (53%)] Loss: 26.130665 Train Epoch: 14 [33280/60000 (55%)] Loss: 25.088482 Train Epoch: 14 [34560/60000 (58%)] Loss: 27.660519 Train Epoch: 14 [35840/60000 (60%)] Loss: 25.195019 Train Epoch: 14 [37120/60000 (62%)] Loss: 24.607042 Train Epoch: 14 [38400/60000 (64%)] Loss: 24.896757 Train Epoch: 14 [39680/60000 (66%)] Loss: 24.338785 Train Epoch: 14 [40960/60000 (68%)] Loss: 26.129469 Train Epoch: 14 [42240/60000 (70%)] Loss: 25.276936 Train Epoch: 14 [43520/60000 (72%)] Loss: 25.923145 Train Epoch: 14 [44800/60000 (75%)] Loss: 24.660961 Train Epoch: 14 [46080/60000 (77%)] Loss: 24.541460 Train Epoch: 14 [47360/60000 (79%)] Loss: 24.351767 Train Epoch: 14 [48640/60000 (81%)] Loss: 25.532055 Train Epoch: 14 [49920/60000 (83%)] Loss: 24.135414 Train Epoch: 14 [51200/60000 (85%)] Loss: 25.250099 Train Epoch: 14 [52480/60000 (87%)] Loss: 26.539398 Train Epoch: 14 [53760/60000 (90%)] Loss: 27.546663 Train Epoch: 14 [55040/60000 (92%)] Loss: 25.064112 Train Epoch: 14 [56320/60000 (94%)] Loss: 25.249399 Train Epoch: 14 [57600/60000 (96%)] Loss: 24.639912 Train Epoch: 14 [58880/60000 (98%)] Loss: 25.213625 ====> Epoch: 14 Average loss: 25.4490 ====> Test set loss: 25.4881 Train Epoch: 15 [0/60000 (0%)] Loss: 25.353418 Train Epoch: 15 [1280/60000 (2%)] Loss: 25.203726 Train Epoch: 15 [2560/60000 (4%)] Loss: 25.334223 Train Epoch: 15 [3840/60000 (6%)] Loss: 26.069750 Train Epoch: 15 [5120/60000 (9%)] Loss: 25.357075 Train Epoch: 15 [6400/60000 (11%)] Loss: 25.081993 Train Epoch: 15 [7680/60000 (13%)] Loss: 24.981255 Train Epoch: 15 [8960/60000 (15%)] Loss: 24.851339 Train Epoch: 15 [10240/60000 (17%)] Loss: 23.932318 Train Epoch: 15 [11520/60000 (19%)] Loss: 25.628338 Train Epoch: 15 [12800/60000 (21%)] Loss: 26.572693 Train Epoch: 15 [14080/60000 (23%)] Loss: 25.780699 Train Epoch: 15 [15360/60000 (26%)] Loss: 24.903286 Train Epoch: 15 [16640/60000 (28%)] Loss: 24.340599 Train Epoch: 15 [17920/60000 (30%)] Loss: 25.169491 Train Epoch: 15 [19200/60000 (32%)] Loss: 25.491375 Train Epoch: 15 [20480/60000 (34%)] Loss: 25.830723 Train Epoch: 15 [21760/60000 (36%)] Loss: 24.751497 Train Epoch: 15 [23040/60000 (38%)] Loss: 26.448723 Train Epoch: 15 [24320/60000 (41%)] Loss: 26.830456 Train Epoch: 15 [25600/60000 (43%)] Loss: 25.871994 Train Epoch: 15 [26880/60000 (45%)] Loss: 26.982229 Train Epoch: 15 [28160/60000 (47%)] Loss: 25.778095 Train Epoch: 15 [29440/60000 (49%)] Loss: 25.305925 Train Epoch: 15 [30720/60000 (51%)] Loss: 25.291811 Train Epoch: 15 [32000/60000 (53%)] Loss: 25.103069 Train Epoch: 15 [33280/60000 (55%)] Loss: 24.432085 Train Epoch: 15 [34560/60000 (58%)] Loss: 24.363420 Train Epoch: 15 [35840/60000 (60%)] Loss: 26.006617 Train Epoch: 15 [37120/60000 (62%)] Loss: 26.898283 Train Epoch: 15 [38400/60000 (64%)] Loss: 25.146957 Train Epoch: 15 [39680/60000 (66%)] Loss: 25.707195 Train Epoch: 15 [40960/60000 (68%)] Loss: 23.616167 Train Epoch: 15 [42240/60000 (70%)] Loss: 25.583942 Train Epoch: 15 [43520/60000 (72%)] Loss: 26.988722 Train Epoch: 15 [44800/60000 (75%)] Loss: 24.519503 Train Epoch: 15 [46080/60000 (77%)] Loss: 27.141891 Train Epoch: 15 [47360/60000 (79%)] Loss: 26.150818 Train Epoch: 15 [48640/60000 (81%)] Loss: 24.304522 Train Epoch: 15 [49920/60000 (83%)] Loss: 25.023785 Train Epoch: 15 [51200/60000 (85%)] Loss: 25.554714 Train Epoch: 15 [52480/60000 (87%)] Loss: 25.606869 Train Epoch: 15 [53760/60000 (90%)] Loss: 24.922077 Train Epoch: 15 [55040/60000 (92%)] Loss: 25.905342 Train Epoch: 15 [56320/60000 (94%)] Loss: 26.072063 Train Epoch: 15 [57600/60000 (96%)] Loss: 25.412306 Train Epoch: 15 [58880/60000 (98%)] Loss: 25.715504 ====> Epoch: 15 Average loss: 25.3495 ====> Test set loss: 25.2984 Train Epoch: 16 [0/60000 (0%)] Loss: 25.287289 Train Epoch: 16 [1280/60000 (2%)] Loss: 25.274588 Train Epoch: 16 [2560/60000 (4%)] Loss: 23.924957 Train Epoch: 16 [3840/60000 (6%)] Loss: 25.006163 Train Epoch: 16 [5120/60000 (9%)] Loss: 25.256260 Train Epoch: 16 [6400/60000 (11%)] Loss: 24.714024 Train Epoch: 16 [7680/60000 (13%)] Loss: 24.849186 Train Epoch: 16 [8960/60000 (15%)] Loss: 26.106911 Train Epoch: 16 [10240/60000 (17%)] Loss: 24.604935 Train Epoch: 16 [11520/60000 (19%)] Loss: 25.046719 Train Epoch: 16 [12800/60000 (21%)] Loss: 25.636272 Train Epoch: 16 [14080/60000 (23%)] Loss: 24.464191 Train Epoch: 16 [15360/60000 (26%)] Loss: 24.760063 Train Epoch: 16 [16640/60000 (28%)] Loss: 26.296886 Train Epoch: 16 [17920/60000 (30%)] Loss: 25.235054 Train Epoch: 16 [19200/60000 (32%)] Loss: 24.024105 Train Epoch: 16 [20480/60000 (34%)] Loss: 25.874323 Train Epoch: 16 [21760/60000 (36%)] Loss: 26.115530 Train Epoch: 16 [23040/60000 (38%)] Loss: 25.654285 Train Epoch: 16 [24320/60000 (41%)] Loss: 25.258589 Train Epoch: 16 [25600/60000 (43%)] Loss: 24.713318 Train Epoch: 16 [26880/60000 (45%)] Loss: 25.611023 Train Epoch: 16 [28160/60000 (47%)] Loss: 25.043491 Train Epoch: 16 [29440/60000 (49%)] Loss: 25.997383 Train Epoch: 16 [30720/60000 (51%)] Loss: 24.199818 Train Epoch: 16 [32000/60000 (53%)] Loss: 24.960106 Train Epoch: 16 [33280/60000 (55%)] Loss: 25.836441 Train Epoch: 16 [34560/60000 (58%)] Loss: 26.230965 Train Epoch: 16 [35840/60000 (60%)] Loss: 24.811510 Train Epoch: 16 [37120/60000 (62%)] Loss: 26.258945 Train Epoch: 16 [38400/60000 (64%)] Loss: 25.453291 Train Epoch: 16 [39680/60000 (66%)] Loss: 24.723392 Train Epoch: 16 [40960/60000 (68%)] Loss: 26.372299 Train Epoch: 16 [42240/60000 (70%)] Loss: 24.394251 Train Epoch: 16 [43520/60000 (72%)] Loss: 26.127829 Train Epoch: 16 [44800/60000 (75%)] Loss: 26.920200 Train Epoch: 16 [46080/60000 (77%)] Loss: 25.193237 Train Epoch: 16 [47360/60000 (79%)] Loss: 25.587238 Train Epoch: 16 [48640/60000 (81%)] Loss: 24.035242 Train Epoch: 16 [49920/60000 (83%)] Loss: 23.657045 Train Epoch: 16 [51200/60000 (85%)] Loss: 26.158449 Train Epoch: 16 [52480/60000 (87%)] Loss: 25.033974 Train Epoch: 16 [53760/60000 (90%)] Loss: 24.262096 Train Epoch: 16 [55040/60000 (92%)] Loss: 24.634947 Train Epoch: 16 [56320/60000 (94%)] Loss: 25.659679 Train Epoch: 16 [57600/60000 (96%)] Loss: 23.071453 Train Epoch: 16 [58880/60000 (98%)] Loss: 24.835300 ====> Epoch: 16 Average loss: 25.2447 ====> Test set loss: 25.2836 Train Epoch: 17 [0/60000 (0%)] Loss: 26.073715 Train Epoch: 17 [1280/60000 (2%)] Loss: 24.135416 Train Epoch: 17 [2560/60000 (4%)] Loss: 25.587368 Train Epoch: 17 [3840/60000 (6%)] Loss: 24.067755 Train Epoch: 17 [5120/60000 (9%)] Loss: 24.620247 Train Epoch: 17 [6400/60000 (11%)] Loss: 23.817074 Train Epoch: 17 [7680/60000 (13%)] Loss: 24.972525 Train Epoch: 17 [8960/60000 (15%)] Loss: 24.050423 Train Epoch: 17 [10240/60000 (17%)] Loss: 24.705055 Train Epoch: 17 [11520/60000 (19%)] Loss: 26.254749 Train Epoch: 17 [12800/60000 (21%)] Loss: 25.567207 Train Epoch: 17 [14080/60000 (23%)] Loss: 24.761154 Train Epoch: 17 [15360/60000 (26%)] Loss: 24.761370 Train Epoch: 17 [16640/60000 (28%)] Loss: 26.314098 Train Epoch: 17 [17920/60000 (30%)] Loss: 25.424147 Train Epoch: 17 [19200/60000 (32%)] Loss: 24.171791 Train Epoch: 17 [20480/60000 (34%)] Loss: 25.248920 Train Epoch: 17 [21760/60000 (36%)] Loss: 23.999594 Train Epoch: 17 [23040/60000 (38%)] Loss: 26.624790 Train Epoch: 17 [24320/60000 (41%)] Loss: 26.179775 Train Epoch: 17 [25600/60000 (43%)] Loss: 24.139301 Train Epoch: 17 [26880/60000 (45%)] Loss: 24.834496 Train Epoch: 17 [28160/60000 (47%)] Loss: 23.548359 Train Epoch: 17 [29440/60000 (49%)] Loss: 24.798496 Train Epoch: 17 [30720/60000 (51%)] Loss: 25.301252 Train Epoch: 17 [32000/60000 (53%)] Loss: 23.754826 Train Epoch: 17 [33280/60000 (55%)] Loss: 24.303001 Train Epoch: 17 [34560/60000 (58%)] Loss: 23.057934 Train Epoch: 17 [35840/60000 (60%)] Loss: 25.668839 Train Epoch: 17 [37120/60000 (62%)] Loss: 25.112968 Train Epoch: 17 [38400/60000 (64%)] Loss: 26.207878 Train Epoch: 17 [39680/60000 (66%)] Loss: 25.345247 Train Epoch: 17 [40960/60000 (68%)] Loss: 25.362926 Train Epoch: 17 [42240/60000 (70%)] Loss: 25.293575 Train Epoch: 17 [43520/60000 (72%)] Loss: 26.949568 Train Epoch: 17 [44800/60000 (75%)] Loss: 25.262838 Train Epoch: 17 [46080/60000 (77%)] Loss: 25.241653 Train Epoch: 17 [47360/60000 (79%)] Loss: 26.383856 Train Epoch: 17 [48640/60000 (81%)] Loss: 24.816317 Train Epoch: 17 [49920/60000 (83%)] Loss: 23.912193 Train Epoch: 17 [51200/60000 (85%)] Loss: 25.108944 Train Epoch: 17 [52480/60000 (87%)] Loss: 24.037033 Train Epoch: 17 [53760/60000 (90%)] Loss: 25.106501 Train Epoch: 17 [55040/60000 (92%)] Loss: 28.013359 Train Epoch: 17 [56320/60000 (94%)] Loss: 23.705534 Train Epoch: 17 [57600/60000 (96%)] Loss: 25.870586 Train Epoch: 17 [58880/60000 (98%)] Loss: 23.839672 ====> Epoch: 17 Average loss: 25.1485 ====> Test set loss: 25.1302 Train Epoch: 18 [0/60000 (0%)] Loss: 24.657639 Train Epoch: 18 [1280/60000 (2%)] Loss: 25.607414 Train Epoch: 18 [2560/60000 (4%)] Loss: 24.603117 Train Epoch: 18 [3840/60000 (6%)] Loss: 26.124979 Train Epoch: 18 [5120/60000 (9%)] Loss: 24.694801 Train Epoch: 18 [6400/60000 (11%)] Loss: 24.920832 Train Epoch: 18 [7680/60000 (13%)] Loss: 25.264971 Train Epoch: 18 [8960/60000 (15%)] Loss: 25.690258 Train Epoch: 18 [10240/60000 (17%)] Loss: 24.486105 Train Epoch: 18 [11520/60000 (19%)] Loss: 24.861473 Train Epoch: 18 [12800/60000 (21%)] Loss: 26.727913 Train Epoch: 18 [14080/60000 (23%)] Loss: 26.860630 Train Epoch: 18 [15360/60000 (26%)] Loss: 25.786547 Train Epoch: 18 [16640/60000 (28%)] Loss: 25.985281 Train Epoch: 18 [17920/60000 (30%)] Loss: 24.683353 Train Epoch: 18 [19200/60000 (32%)] Loss: 25.990423 Train Epoch: 18 [20480/60000 (34%)] Loss: 24.559011 Train Epoch: 18 [21760/60000 (36%)] Loss: 25.728617 Train Epoch: 18 [23040/60000 (38%)] Loss: 24.158039 Train Epoch: 18 [24320/60000 (41%)] Loss: 26.173542 Train Epoch: 18 [25600/60000 (43%)] Loss: 25.955688 Train Epoch: 18 [26880/60000 (45%)] Loss: 24.824532 Train Epoch: 18 [28160/60000 (47%)] Loss: 24.207827 Train Epoch: 18 [29440/60000 (49%)] Loss: 25.296551 Train Epoch: 18 [30720/60000 (51%)] Loss: 26.591213 Train Epoch: 18 [32000/60000 (53%)] Loss: 26.577703 Train Epoch: 18 [33280/60000 (55%)] Loss: 24.231333 Train Epoch: 18 [34560/60000 (58%)] Loss: 24.708935 Train Epoch: 18 [35840/60000 (60%)] Loss: 25.119032 Train Epoch: 18 [37120/60000 (62%)] Loss: 24.391623 Train Epoch: 18 [38400/60000 (64%)] Loss: 24.737677 Train Epoch: 18 [39680/60000 (66%)] Loss: 25.011330 Train Epoch: 18 [40960/60000 (68%)] Loss: 24.493496 Train Epoch: 18 [42240/60000 (70%)] Loss: 25.789186 Train Epoch: 18 [43520/60000 (72%)] Loss: 25.000275 Train Epoch: 18 [44800/60000 (75%)] Loss: 24.469414 Train Epoch: 18 [46080/60000 (77%)] Loss: 25.248484 Train Epoch: 18 [47360/60000 (79%)] Loss: 23.194241 Train Epoch: 18 [48640/60000 (81%)] Loss: 27.314930 Train Epoch: 18 [49920/60000 (83%)] Loss: 24.577200 Train Epoch: 18 [51200/60000 (85%)] Loss: 24.563530 Train Epoch: 18 [52480/60000 (87%)] Loss: 24.580931 Train Epoch: 18 [53760/60000 (90%)] Loss: 24.417870 Train Epoch: 18 [55040/60000 (92%)] Loss: 24.506889 Train Epoch: 18 [56320/60000 (94%)] Loss: 24.058184 Train Epoch: 18 [57600/60000 (96%)] Loss: 25.189716 Train Epoch: 18 [58880/60000 (98%)] Loss: 24.465206 ====> Epoch: 18 Average loss: 25.0814 ====> Test set loss: 25.2087 Train Epoch: 19 [0/60000 (0%)] Loss: 25.605373 Train Epoch: 19 [1280/60000 (2%)] Loss: 24.986231 Train Epoch: 19 [2560/60000 (4%)] Loss: 24.945940 Train Epoch: 19 [3840/60000 (6%)] Loss: 24.739246 Train Epoch: 19 [5120/60000 (9%)] Loss: 25.016228 Train Epoch: 19 [6400/60000 (11%)] Loss: 25.375965 Train Epoch: 19 [7680/60000 (13%)] Loss: 24.648064 Train Epoch: 19 [8960/60000 (15%)] Loss: 25.811871 Train Epoch: 19 [10240/60000 (17%)] Loss: 25.331360 Train Epoch: 19 [11520/60000 (19%)] Loss: 26.654831 Train Epoch: 19 [12800/60000 (21%)] Loss: 25.350721 Train Epoch: 19 [14080/60000 (23%)] Loss: 25.222300 Train Epoch: 19 [15360/60000 (26%)] Loss: 25.698532 Train Epoch: 19 [16640/60000 (28%)] Loss: 25.882296 Train Epoch: 19 [17920/60000 (30%)] Loss: 25.252605 Train Epoch: 19 [19200/60000 (32%)] Loss: 25.672401 Train Epoch: 19 [20480/60000 (34%)] Loss: 24.418949 Train Epoch: 19 [21760/60000 (36%)] Loss: 24.783655 Train Epoch: 19 [23040/60000 (38%)] Loss: 25.156004 Train Epoch: 19 [24320/60000 (41%)] Loss: 25.323395 Train Epoch: 19 [25600/60000 (43%)] Loss: 25.140112 Train Epoch: 19 [26880/60000 (45%)] Loss: 25.904882 Train Epoch: 19 [28160/60000 (47%)] Loss: 24.554319 Train Epoch: 19 [29440/60000 (49%)] Loss: 23.754276 Train Epoch: 19 [30720/60000 (51%)] Loss: 22.921015 Train Epoch: 19 [32000/60000 (53%)] Loss: 25.575834 Train Epoch: 19 [33280/60000 (55%)] Loss: 25.600025 Train Epoch: 19 [34560/60000 (58%)] Loss: 24.088894 Train Epoch: 19 [35840/60000 (60%)] Loss: 23.863861 Train Epoch: 19 [37120/60000 (62%)] Loss: 25.201357 Train Epoch: 19 [38400/60000 (64%)] Loss: 24.132179 Train Epoch: 19 [39680/60000 (66%)] Loss: 24.151466 Train Epoch: 19 [40960/60000 (68%)] Loss: 24.409397 Train Epoch: 19 [42240/60000 (70%)] Loss: 25.576826 Train Epoch: 19 [43520/60000 (72%)] Loss: 24.785027 Train Epoch: 19 [44800/60000 (75%)] Loss: 25.997990 Train Epoch: 19 [46080/60000 (77%)] Loss: 25.127090 Train Epoch: 19 [47360/60000 (79%)] Loss: 26.556269 Train Epoch: 19 [48640/60000 (81%)] Loss: 24.532322 Train Epoch: 19 [49920/60000 (83%)] Loss: 25.520502 Train Epoch: 19 [51200/60000 (85%)] Loss: 25.147854 Train Epoch: 19 [52480/60000 (87%)] Loss: 24.834476 Train Epoch: 19 [53760/60000 (90%)] Loss: 25.349037 Train Epoch: 19 [55040/60000 (92%)] Loss: 25.851921 Train Epoch: 19 [56320/60000 (94%)] Loss: 26.107565 Train Epoch: 19 [57600/60000 (96%)] Loss: 24.848471 Train Epoch: 19 [58880/60000 (98%)] Loss: 27.260666 ====> Epoch: 19 Average loss: 24.9946 ====> Test set loss: 24.9750 Train Epoch: 20 [0/60000 (0%)] Loss: 25.310549 Train Epoch: 20 [1280/60000 (2%)] Loss: 24.320076 Train Epoch: 20 [2560/60000 (4%)] Loss: 24.937870 Train Epoch: 20 [3840/60000 (6%)] Loss: 24.616074 Train Epoch: 20 [5120/60000 (9%)] Loss: 26.862352 Train Epoch: 20 [6400/60000 (11%)] Loss: 24.302101 Train Epoch: 20 [7680/60000 (13%)] Loss: 23.482742 Train Epoch: 20 [8960/60000 (15%)] Loss: 25.271820 Train Epoch: 20 [10240/60000 (17%)] Loss: 26.227955 Train Epoch: 20 [11520/60000 (19%)] Loss: 24.375500 Train Epoch: 20 [12800/60000 (21%)] Loss: 26.758331 Train Epoch: 20 [14080/60000 (23%)] Loss: 25.646357 Train Epoch: 20 [15360/60000 (26%)] Loss: 23.892586 Train Epoch: 20 [16640/60000 (28%)] Loss: 25.184803 Train Epoch: 20 [17920/60000 (30%)] Loss: 24.180014 Train Epoch: 20 [19200/60000 (32%)] Loss: 26.533409 Train Epoch: 20 [20480/60000 (34%)] Loss: 26.121521 Train Epoch: 20 [21760/60000 (36%)] Loss: 26.429501 Train Epoch: 20 [23040/60000 (38%)] Loss: 24.798325 Train Epoch: 20 [24320/60000 (41%)] Loss: 25.594061 Train Epoch: 20 [25600/60000 (43%)] Loss: 22.840010 Train Epoch: 20 [26880/60000 (45%)] Loss: 24.644701 Train Epoch: 20 [28160/60000 (47%)] Loss: 25.719961 Train Epoch: 20 [29440/60000 (49%)] Loss: 24.513191 Train Epoch: 20 [30720/60000 (51%)] Loss: 25.019405 Train Epoch: 20 [32000/60000 (53%)] Loss: 26.492428 Train Epoch: 20 [33280/60000 (55%)] Loss: 24.208311 Train Epoch: 20 [34560/60000 (58%)] Loss: 24.322302 Train Epoch: 20 [35840/60000 (60%)] Loss: 24.421446 Train Epoch: 20 [37120/60000 (62%)] Loss: 24.680313 Train Epoch: 20 [38400/60000 (64%)] Loss: 23.834406 Train Epoch: 20 [39680/60000 (66%)] Loss: 25.679783 Train Epoch: 20 [40960/60000 (68%)] Loss: 26.314329 Train Epoch: 20 [42240/60000 (70%)] Loss: 24.670841 Train Epoch: 20 [43520/60000 (72%)] Loss: 25.000813 Train Epoch: 20 [44800/60000 (75%)] Loss: 25.249430 Train Epoch: 20 [46080/60000 (77%)] Loss: 24.767944 Train Epoch: 20 [47360/60000 (79%)] Loss: 25.769028 Train Epoch: 20 [48640/60000 (81%)] Loss: 24.520130 Train Epoch: 20 [49920/60000 (83%)] Loss: 24.255692 Train Epoch: 20 [51200/60000 (85%)] Loss: 25.015228 Train Epoch: 20 [52480/60000 (87%)] Loss: 24.699341 Train Epoch: 20 [53760/60000 (90%)] Loss: 24.070126 Train Epoch: 20 [55040/60000 (92%)] Loss: 24.245720 Train Epoch: 20 [56320/60000 (94%)] Loss: 22.886745 Train Epoch: 20 [57600/60000 (96%)] Loss: 25.043488 Train Epoch: 20 [58880/60000 (98%)] Loss: 24.425570 ====> Epoch: 20 Average loss: 24.9345 ====> Test set loss: 24.9439 Train Epoch: 21 [0/60000 (0%)] Loss: 25.326077 Train Epoch: 21 [1280/60000 (2%)] Loss: 25.027363 Train Epoch: 21 [2560/60000 (4%)] Loss: 25.452412 Train Epoch: 21 [3840/60000 (6%)] Loss: 25.857521 Train Epoch: 21 [5120/60000 (9%)] Loss: 26.149096 Train Epoch: 21 [6400/60000 (11%)] Loss: 26.130024 Train Epoch: 21 [7680/60000 (13%)] Loss: 24.317492 Train Epoch: 21 [8960/60000 (15%)] Loss: 26.569529 Train Epoch: 21 [10240/60000 (17%)] Loss: 22.860222 Train Epoch: 21 [11520/60000 (19%)] Loss: 24.854858 Train Epoch: 21 [12800/60000 (21%)] Loss: 25.231014 Train Epoch: 21 [14080/60000 (23%)] Loss: 25.627867 Train Epoch: 21 [15360/60000 (26%)] Loss: 23.723866 Train Epoch: 21 [16640/60000 (28%)] Loss: 25.972727 Train Epoch: 21 [17920/60000 (30%)] Loss: 23.772007 Train Epoch: 21 [19200/60000 (32%)] Loss: 25.549143 Train Epoch: 21 [20480/60000 (34%)] Loss: 24.355705 Train Epoch: 21 [21760/60000 (36%)] Loss: 24.681484 Train Epoch: 21 [23040/60000 (38%)] Loss: 25.256130 Train Epoch: 21 [24320/60000 (41%)] Loss: 23.903763 Train Epoch: 21 [25600/60000 (43%)] Loss: 24.032749 Train Epoch: 21 [26880/60000 (45%)] Loss: 25.594763 Train Epoch: 21 [28160/60000 (47%)] Loss: 25.063757 Train Epoch: 21 [29440/60000 (49%)] Loss: 25.363710 Train Epoch: 21 [30720/60000 (51%)] Loss: 24.855581 Train Epoch: 21 [32000/60000 (53%)] Loss: 24.554146 Train Epoch: 21 [33280/60000 (55%)] Loss: 25.042288 Train Epoch: 21 [34560/60000 (58%)] Loss: 24.192558 Train Epoch: 21 [35840/60000 (60%)] Loss: 24.480854 Train Epoch: 21 [37120/60000 (62%)] Loss: 24.719851 Train Epoch: 21 [38400/60000 (64%)] Loss: 25.449879 Train Epoch: 21 [39680/60000 (66%)] Loss: 24.791027 Train Epoch: 21 [40960/60000 (68%)] Loss: 24.534588 Train Epoch: 21 [42240/60000 (70%)] Loss: 26.395973 Train Epoch: 21 [43520/60000 (72%)] Loss: 25.262186 Train Epoch: 21 [44800/60000 (75%)] Loss: 24.504738 Train Epoch: 21 [46080/60000 (77%)] Loss: 25.280968 Train Epoch: 21 [47360/60000 (79%)] Loss: 23.278709 Train Epoch: 21 [48640/60000 (81%)] Loss: 25.900259 Train Epoch: 21 [49920/60000 (83%)] Loss: 25.518932 Train Epoch: 21 [51200/60000 (85%)] Loss: 26.179293 Train Epoch: 21 [52480/60000 (87%)] Loss: 23.497372 Train Epoch: 21 [53760/60000 (90%)] Loss: 24.366714 Train Epoch: 21 [55040/60000 (92%)] Loss: 25.651653 Train Epoch: 21 [56320/60000 (94%)] Loss: 26.317650 Train Epoch: 21 [57600/60000 (96%)] Loss: 24.386562 Train Epoch: 21 [58880/60000 (98%)] Loss: 24.275213 ====> Epoch: 21 Average loss: 24.8304 ====> Test set loss: 25.0507 Train Epoch: 22 [0/60000 (0%)] Loss: 26.303423 Train Epoch: 22 [1280/60000 (2%)] Loss: 25.569546 Train Epoch: 22 [2560/60000 (4%)] Loss: 23.786173 Train Epoch: 22 [3840/60000 (6%)] Loss: 25.588867 Train Epoch: 22 [5120/60000 (9%)] Loss: 24.555660 Train Epoch: 22 [6400/60000 (11%)] Loss: 24.634064 Train Epoch: 22 [7680/60000 (13%)] Loss: 25.251596 Train Epoch: 22 [8960/60000 (15%)] Loss: 24.901985 Train Epoch: 22 [10240/60000 (17%)] Loss: 24.352676 Train Epoch: 22 [11520/60000 (19%)] Loss: 25.327492 Train Epoch: 22 [12800/60000 (21%)] Loss: 26.017918 Train Epoch: 22 [14080/60000 (23%)] Loss: 25.013756 Train Epoch: 22 [15360/60000 (26%)] Loss: 24.714214 Train Epoch: 22 [16640/60000 (28%)] Loss: 23.958450 Train Epoch: 22 [17920/60000 (30%)] Loss: 23.635551 Train Epoch: 22 [19200/60000 (32%)] Loss: 25.313679 Train Epoch: 22 [20480/60000 (34%)] Loss: 23.563484 Train Epoch: 22 [21760/60000 (36%)] Loss: 23.843466 Train Epoch: 22 [23040/60000 (38%)] Loss: 23.576855 Train Epoch: 22 [24320/60000 (41%)] Loss: 25.416370 Train Epoch: 22 [25600/60000 (43%)] Loss: 24.241383 Train Epoch: 22 [26880/60000 (45%)] Loss: 23.999165 Train Epoch: 22 [28160/60000 (47%)] Loss: 25.905735 Train Epoch: 22 [29440/60000 (49%)] Loss: 24.245508 Train Epoch: 22 [30720/60000 (51%)] Loss: 24.720182 Train Epoch: 22 [32000/60000 (53%)] Loss: 24.501247 Train Epoch: 22 [33280/60000 (55%)] Loss: 24.929674 Train Epoch: 22 [34560/60000 (58%)] Loss: 24.923401 Train Epoch: 22 [35840/60000 (60%)] Loss: 23.753483 Train Epoch: 22 [37120/60000 (62%)] Loss: 23.639904 Train Epoch: 22 [38400/60000 (64%)] Loss: 24.495644 Train Epoch: 22 [39680/60000 (66%)] Loss: 24.174978 Train Epoch: 22 [40960/60000 (68%)] Loss: 25.270973 Train Epoch: 22 [42240/60000 (70%)] Loss: 24.948330 Train Epoch: 22 [43520/60000 (72%)] Loss: 24.925734 Train Epoch: 22 [44800/60000 (75%)] Loss: 24.851364 Train Epoch: 22 [46080/60000 (77%)] Loss: 24.453382 Train Epoch: 22 [47360/60000 (79%)] Loss: 25.979858 Train Epoch: 22 [48640/60000 (81%)] Loss: 26.142975 Train Epoch: 22 [49920/60000 (83%)] Loss: 26.691704 Train Epoch: 22 [51200/60000 (85%)] Loss: 24.617329 Train Epoch: 22 [52480/60000 (87%)] Loss: 23.714348 Train Epoch: 22 [53760/60000 (90%)] Loss: 24.750940 Train Epoch: 22 [55040/60000 (92%)] Loss: 26.429619 Train Epoch: 22 [56320/60000 (94%)] Loss: 24.911911 Train Epoch: 22 [57600/60000 (96%)] Loss: 24.724964 Train Epoch: 22 [58880/60000 (98%)] Loss: 25.493515 ====> Epoch: 22 Average loss: 24.8079 ====> Test set loss: 24.7715 Train Epoch: 23 [0/60000 (0%)] Loss: 24.880976 Train Epoch: 23 [1280/60000 (2%)] Loss: 24.267244 Train Epoch: 23 [2560/60000 (4%)] Loss: 24.127716 Train Epoch: 23 [3840/60000 (6%)] Loss: 25.414312 Train Epoch: 23 [5120/60000 (9%)] Loss: 24.567574 Train Epoch: 23 [6400/60000 (11%)] Loss: 23.890333 Train Epoch: 23 [7680/60000 (13%)] Loss: 22.676268 Train Epoch: 23 [8960/60000 (15%)] Loss: 24.834116 Train Epoch: 23 [10240/60000 (17%)] Loss: 25.368698 Train Epoch: 23 [11520/60000 (19%)] Loss: 24.413834 Train Epoch: 23 [12800/60000 (21%)] Loss: 27.055244 Train Epoch: 23 [14080/60000 (23%)] Loss: 24.602003 Train Epoch: 23 [15360/60000 (26%)] Loss: 26.611826 Train Epoch: 23 [16640/60000 (28%)] Loss: 24.472115 Train Epoch: 23 [17920/60000 (30%)] Loss: 23.714386 Train Epoch: 23 [19200/60000 (32%)] Loss: 24.160511 Train Epoch: 23 [20480/60000 (34%)] Loss: 26.346912 Train Epoch: 23 [21760/60000 (36%)] Loss: 24.091698 Train Epoch: 23 [23040/60000 (38%)] Loss: 24.921970 Train Epoch: 23 [24320/60000 (41%)] Loss: 23.432178 Train Epoch: 23 [25600/60000 (43%)] Loss: 25.205561 Train Epoch: 23 [26880/60000 (45%)] Loss: 25.921967 Train Epoch: 23 [28160/60000 (47%)] Loss: 24.379524 Train Epoch: 23 [29440/60000 (49%)] Loss: 24.637623 Train Epoch: 23 [30720/60000 (51%)] Loss: 23.566734 Train Epoch: 23 [32000/60000 (53%)] Loss: 24.935905 Train Epoch: 23 [33280/60000 (55%)] Loss: 23.070694 Train Epoch: 23 [34560/60000 (58%)] Loss: 24.807171 Train Epoch: 23 [35840/60000 (60%)] Loss: 23.702314 Train Epoch: 23 [37120/60000 (62%)] Loss: 24.479197 Train Epoch: 23 [38400/60000 (64%)] Loss: 26.940529 Train Epoch: 23 [39680/60000 (66%)] Loss: 25.886105 Train Epoch: 23 [40960/60000 (68%)] Loss: 24.902557 Train Epoch: 23 [42240/60000 (70%)] Loss: 24.683584 Train Epoch: 23 [43520/60000 (72%)] Loss: 25.387358 Train Epoch: 23 [44800/60000 (75%)] Loss: 24.367096 Train Epoch: 23 [46080/60000 (77%)] Loss: 25.443676 Train Epoch: 23 [47360/60000 (79%)] Loss: 25.985737 Train Epoch: 23 [48640/60000 (81%)] Loss: 25.004768 Train Epoch: 23 [49920/60000 (83%)] Loss: 23.865978 Train Epoch: 23 [51200/60000 (85%)] Loss: 24.461582 Train Epoch: 23 [52480/60000 (87%)] Loss: 23.495934 Train Epoch: 23 [53760/60000 (90%)] Loss: 25.872965 Train Epoch: 23 [55040/60000 (92%)] Loss: 24.887434 Train Epoch: 23 [56320/60000 (94%)] Loss: 25.031961 Train Epoch: 23 [57600/60000 (96%)] Loss: 24.137886 Train Epoch: 23 [58880/60000 (98%)] Loss: 24.109699 ====> Epoch: 23 Average loss: 24.7229 ====> Test set loss: 24.6755 Train Epoch: 24 [0/60000 (0%)] Loss: 23.433109 Train Epoch: 24 [1280/60000 (2%)] Loss: 25.200195 Train Epoch: 24 [2560/60000 (4%)] Loss: 24.694958 Train Epoch: 24 [3840/60000 (6%)] Loss: 24.510126 Train Epoch: 24 [5120/60000 (9%)] Loss: 25.716139 Train Epoch: 24 [6400/60000 (11%)] Loss: 25.316860 Train Epoch: 24 [7680/60000 (13%)] Loss: 23.710648 Train Epoch: 24 [8960/60000 (15%)] Loss: 24.463100 Train Epoch: 24 [10240/60000 (17%)] Loss: 24.617439 Train Epoch: 24 [11520/60000 (19%)] Loss: 23.848993 Train Epoch: 24 [12800/60000 (21%)] Loss: 23.061731 Train Epoch: 24 [14080/60000 (23%)] Loss: 24.760513 Train Epoch: 24 [15360/60000 (26%)] Loss: 25.082256 Train Epoch: 24 [16640/60000 (28%)] Loss: 23.235920 Train Epoch: 24 [17920/60000 (30%)] Loss: 23.429438 Train Epoch: 24 [19200/60000 (32%)] Loss: 24.860186 Train Epoch: 24 [20480/60000 (34%)] Loss: 23.957548 Train Epoch: 24 [21760/60000 (36%)] Loss: 25.292519 Train Epoch: 24 [23040/60000 (38%)] Loss: 25.954372 Train Epoch: 24 [24320/60000 (41%)] Loss: 24.714584 Train Epoch: 24 [25600/60000 (43%)] Loss: 24.992275 Train Epoch: 24 [26880/60000 (45%)] Loss: 24.442137 Train Epoch: 24 [28160/60000 (47%)] Loss: 24.266296 Train Epoch: 24 [29440/60000 (49%)] Loss: 24.479130 Train Epoch: 24 [30720/60000 (51%)] Loss: 25.466003 Train Epoch: 24 [32000/60000 (53%)] Loss: 25.258305 Train Epoch: 24 [33280/60000 (55%)] Loss: 25.682522 Train Epoch: 24 [34560/60000 (58%)] Loss: 25.187368 Train Epoch: 24 [35840/60000 (60%)] Loss: 25.566868 Train Epoch: 24 [37120/60000 (62%)] Loss: 24.040916 Train Epoch: 24 [38400/60000 (64%)] Loss: 24.423105 Train Epoch: 24 [39680/60000 (66%)] Loss: 23.603233 Train Epoch: 24 [40960/60000 (68%)] Loss: 24.923225 Train Epoch: 24 [42240/60000 (70%)] Loss: 25.146803 Train Epoch: 24 [43520/60000 (72%)] Loss: 24.380116 Train Epoch: 24 [44800/60000 (75%)] Loss: 23.772003 Train Epoch: 24 [46080/60000 (77%)] Loss: 24.236298 Train Epoch: 24 [47360/60000 (79%)] Loss: 24.317884 Train Epoch: 24 [48640/60000 (81%)] Loss: 23.922594 Train Epoch: 24 [49920/60000 (83%)] Loss: 23.915565 Train Epoch: 24 [51200/60000 (85%)] Loss: 24.826340 Train Epoch: 24 [52480/60000 (87%)] Loss: 24.416172 Train Epoch: 24 [53760/60000 (90%)] Loss: 25.589741 Train Epoch: 24 [55040/60000 (92%)] Loss: 25.366203 Train Epoch: 24 [56320/60000 (94%)] Loss: 24.722967 Train Epoch: 24 [57600/60000 (96%)] Loss: 25.050323 Train Epoch: 24 [58880/60000 (98%)] Loss: 23.404961 ====> Epoch: 24 Average loss: 24.6452 ====> Test set loss: 24.7468 Train Epoch: 25 [0/60000 (0%)] Loss: 25.396702 Train Epoch: 25 [1280/60000 (2%)] Loss: 23.822971 Train Epoch: 25 [2560/60000 (4%)] Loss: 25.606462 Train Epoch: 25 [3840/60000 (6%)] Loss: 25.238754 Train Epoch: 25 [5120/60000 (9%)] Loss: 25.160618 Train Epoch: 25 [6400/60000 (11%)] Loss: 24.664227 Train Epoch: 25 [7680/60000 (13%)] Loss: 23.420210 Train Epoch: 25 [8960/60000 (15%)] Loss: 24.637566 Train Epoch: 25 [10240/60000 (17%)] Loss: 23.943794 Train Epoch: 25 [11520/60000 (19%)] Loss: 24.436710 Train Epoch: 25 [12800/60000 (21%)] Loss: 24.686436 Train Epoch: 25 [14080/60000 (23%)] Loss: 25.452816 Train Epoch: 25 [15360/60000 (26%)] Loss: 23.953861 Train Epoch: 25 [16640/60000 (28%)] Loss: 25.302074 Train Epoch: 25 [17920/60000 (30%)] Loss: 22.668844 Train Epoch: 25 [19200/60000 (32%)] Loss: 25.751316 Train Epoch: 25 [20480/60000 (34%)] Loss: 24.569963 Train Epoch: 25 [21760/60000 (36%)] Loss: 24.421970 Train Epoch: 25 [23040/60000 (38%)] Loss: 24.099541 Train Epoch: 25 [24320/60000 (41%)] Loss: 24.433094 Train Epoch: 25 [25600/60000 (43%)] Loss: 24.654312 Train Epoch: 25 [26880/60000 (45%)] Loss: 24.165756 Train Epoch: 25 [28160/60000 (47%)] Loss: 25.215683 Train Epoch: 25 [29440/60000 (49%)] Loss: 23.985973 Train Epoch: 25 [30720/60000 (51%)] Loss: 24.354122 Train Epoch: 25 [32000/60000 (53%)] Loss: 25.976141 Train Epoch: 25 [33280/60000 (55%)] Loss: 24.692524 Train Epoch: 25 [34560/60000 (58%)] Loss: 24.979340 Train Epoch: 25 [35840/60000 (60%)] Loss: 25.628078 Train Epoch: 25 [37120/60000 (62%)] Loss: 24.585749 Train Epoch: 25 [38400/60000 (64%)] Loss: 25.711239 Train Epoch: 25 [39680/60000 (66%)] Loss: 24.897636 Train Epoch: 25 [40960/60000 (68%)] Loss: 23.897629 Train Epoch: 25 [42240/60000 (70%)] Loss: 23.689970 Train Epoch: 25 [43520/60000 (72%)] Loss: 24.302441 Train Epoch: 25 [44800/60000 (75%)] Loss: 24.310854 Train Epoch: 25 [46080/60000 (77%)] Loss: 23.943192 Train Epoch: 25 [47360/60000 (79%)] Loss: 24.788591 Train Epoch: 25 [48640/60000 (81%)] Loss: 26.296961 Train Epoch: 25 [49920/60000 (83%)] Loss: 23.709583 Train Epoch: 25 [51200/60000 (85%)] Loss: 25.402075 Train Epoch: 25 [52480/60000 (87%)] Loss: 26.948868 Train Epoch: 25 [53760/60000 (90%)] Loss: 25.052639 Train Epoch: 25 [55040/60000 (92%)] Loss: 25.495955 Train Epoch: 25 [56320/60000 (94%)] Loss: 24.652315 Train Epoch: 25 [57600/60000 (96%)] Loss: 25.215834 Train Epoch: 25 [58880/60000 (98%)] Loss: 25.165394 ====> Epoch: 25 Average loss: 24.5949 ====> Test set loss: 24.6427 Train Epoch: 26 [0/60000 (0%)] Loss: 23.356087 Train Epoch: 26 [1280/60000 (2%)] Loss: 24.355793 Train Epoch: 26 [2560/60000 (4%)] Loss: 25.628016 Train Epoch: 26 [3840/60000 (6%)] Loss: 22.954725 Train Epoch: 26 [5120/60000 (9%)] Loss: 25.027821 Train Epoch: 26 [6400/60000 (11%)] Loss: 24.715651 Train Epoch: 26 [7680/60000 (13%)] Loss: 23.427305 Train Epoch: 26 [8960/60000 (15%)] Loss: 24.436739 Train Epoch: 26 [10240/60000 (17%)] Loss: 25.144541 Train Epoch: 26 [11520/60000 (19%)] Loss: 24.428246 Train Epoch: 26 [12800/60000 (21%)] Loss: 25.219463 Train Epoch: 26 [14080/60000 (23%)] Loss: 26.441685 Train Epoch: 26 [15360/60000 (26%)] Loss: 24.002148 Train Epoch: 26 [16640/60000 (28%)] Loss: 23.763288 Train Epoch: 26 [17920/60000 (30%)] Loss: 24.844776 Train Epoch: 26 [19200/60000 (32%)] Loss: 24.775908 Train Epoch: 26 [20480/60000 (34%)] Loss: 24.074890 Train Epoch: 26 [21760/60000 (36%)] Loss: 24.226023 Train Epoch: 26 [23040/60000 (38%)] Loss: 25.006809 Train Epoch: 26 [24320/60000 (41%)] Loss: 24.866154 Train Epoch: 26 [25600/60000 (43%)] Loss: 26.467869 Train Epoch: 26 [26880/60000 (45%)] Loss: 23.935167 Train Epoch: 26 [28160/60000 (47%)] Loss: 23.705814 Train Epoch: 26 [29440/60000 (49%)] Loss: 23.900860 Train Epoch: 26 [30720/60000 (51%)] Loss: 24.283047 Train Epoch: 26 [32000/60000 (53%)] Loss: 24.195793 Train Epoch: 26 [33280/60000 (55%)] Loss: 25.181007 Train Epoch: 26 [34560/60000 (58%)] Loss: 25.064638 Train Epoch: 26 [35840/60000 (60%)] Loss: 23.708160 Train Epoch: 26 [37120/60000 (62%)] Loss: 23.853619 Train Epoch: 26 [38400/60000 (64%)] Loss: 24.818457 Train Epoch: 26 [39680/60000 (66%)] Loss: 23.902599 Train Epoch: 26 [40960/60000 (68%)] Loss: 23.879200 Train Epoch: 26 [42240/60000 (70%)] Loss: 24.658819 Train Epoch: 26 [43520/60000 (72%)] Loss: 25.026237 Train Epoch: 26 [44800/60000 (75%)] Loss: 24.739658 Train Epoch: 26 [46080/60000 (77%)] Loss: 25.366570 Train Epoch: 26 [47360/60000 (79%)] Loss: 25.386869 Train Epoch: 26 [48640/60000 (81%)] Loss: 24.890141 Train Epoch: 26 [49920/60000 (83%)] Loss: 24.505787 Train Epoch: 26 [51200/60000 (85%)] Loss: 24.268009 Train Epoch: 26 [52480/60000 (87%)] Loss: 24.705072 Train Epoch: 26 [53760/60000 (90%)] Loss: 23.817398 Train Epoch: 26 [55040/60000 (92%)] Loss: 25.262222 Train Epoch: 26 [56320/60000 (94%)] Loss: 24.873604 Train Epoch: 26 [57600/60000 (96%)] Loss: 24.674492 Train Epoch: 26 [58880/60000 (98%)] Loss: 23.752455 ====> Epoch: 26 Average loss: 24.5643 ====> Test set loss: 24.5758 Train Epoch: 27 [0/60000 (0%)] Loss: 24.650307 Train Epoch: 27 [1280/60000 (2%)] Loss: 24.617182 Train Epoch: 27 [2560/60000 (4%)] Loss: 24.161352 Train Epoch: 27 [3840/60000 (6%)] Loss: 25.617712 Train Epoch: 27 [5120/60000 (9%)] Loss: 22.348307 Train Epoch: 27 [6400/60000 (11%)] Loss: 24.108698 Train Epoch: 27 [7680/60000 (13%)] Loss: 23.742184 Train Epoch: 27 [8960/60000 (15%)] Loss: 24.157917 Train Epoch: 27 [10240/60000 (17%)] Loss: 23.403231 Train Epoch: 27 [11520/60000 (19%)] Loss: 24.719963 Train Epoch: 27 [12800/60000 (21%)] Loss: 24.354006 Train Epoch: 27 [14080/60000 (23%)] Loss: 25.386778 Train Epoch: 27 [15360/60000 (26%)] Loss: 25.395025 Train Epoch: 27 [16640/60000 (28%)] Loss: 24.844810 Train Epoch: 27 [17920/60000 (30%)] Loss: 24.041733 Train Epoch: 27 [19200/60000 (32%)] Loss: 24.617693 Train Epoch: 27 [20480/60000 (34%)] Loss: 22.374720 Train Epoch: 27 [21760/60000 (36%)] Loss: 24.584763 Train Epoch: 27 [23040/60000 (38%)] Loss: 25.476585 Train Epoch: 27 [24320/60000 (41%)] Loss: 24.209599 Train Epoch: 27 [25600/60000 (43%)] Loss: 24.835955 Train Epoch: 27 [26880/60000 (45%)] Loss: 24.968277 Train Epoch: 27 [28160/60000 (47%)] Loss: 24.839996 Train Epoch: 27 [29440/60000 (49%)] Loss: 23.135983 Train Epoch: 27 [30720/60000 (51%)] Loss: 25.544683 Train Epoch: 27 [32000/60000 (53%)] Loss: 24.568483 Train Epoch: 27 [33280/60000 (55%)] Loss: 23.911381 Train Epoch: 27 [34560/60000 (58%)] Loss: 24.189518 Train Epoch: 27 [35840/60000 (60%)] Loss: 24.964794 Train Epoch: 27 [37120/60000 (62%)] Loss: 24.375866 Train Epoch: 27 [38400/60000 (64%)] Loss: 23.897106 Train Epoch: 27 [39680/60000 (66%)] Loss: 24.675758 Train Epoch: 27 [40960/60000 (68%)] Loss: 24.682594 Train Epoch: 27 [42240/60000 (70%)] Loss: 23.074913 Train Epoch: 27 [43520/60000 (72%)] Loss: 24.882547 Train Epoch: 27 [44800/60000 (75%)] Loss: 24.927719 Train Epoch: 27 [46080/60000 (77%)] Loss: 23.727028 Train Epoch: 27 [47360/60000 (79%)] Loss: 23.614897 Train Epoch: 27 [48640/60000 (81%)] Loss: 24.121300 Train Epoch: 27 [49920/60000 (83%)] Loss: 24.780998 Train Epoch: 27 [51200/60000 (85%)] Loss: 26.048965 Train Epoch: 27 [52480/60000 (87%)] Loss: 23.656105 Train Epoch: 27 [53760/60000 (90%)] Loss: 23.728977 Train Epoch: 27 [55040/60000 (92%)] Loss: 25.596748 Train Epoch: 27 [56320/60000 (94%)] Loss: 24.076260 Train Epoch: 27 [57600/60000 (96%)] Loss: 25.596466 Train Epoch: 27 [58880/60000 (98%)] Loss: 24.248119 ====> Epoch: 27 Average loss: 24.5164 ====> Test set loss: 24.5443 Train Epoch: 28 [0/60000 (0%)] Loss: 24.677517 Train Epoch: 28 [1280/60000 (2%)] Loss: 23.579874 Train Epoch: 28 [2560/60000 (4%)] Loss: 23.626095 Train Epoch: 28 [3840/60000 (6%)] Loss: 25.669390 Train Epoch: 28 [5120/60000 (9%)] Loss: 24.422098 Train Epoch: 28 [6400/60000 (11%)] Loss: 24.758331 Train Epoch: 28 [7680/60000 (13%)] Loss: 24.956768 Train Epoch: 28 [8960/60000 (15%)] Loss: 23.468960 Train Epoch: 28 [10240/60000 (17%)] Loss: 25.515907 Train Epoch: 28 [11520/60000 (19%)] Loss: 25.569818 Train Epoch: 28 [12800/60000 (21%)] Loss: 25.240913 Train Epoch: 28 [14080/60000 (23%)] Loss: 22.999535 Train Epoch: 28 [15360/60000 (26%)] Loss: 24.273293 Train Epoch: 28 [16640/60000 (28%)] Loss: 23.988945 Train Epoch: 28 [17920/60000 (30%)] Loss: 23.339817 Train Epoch: 28 [19200/60000 (32%)] Loss: 23.773216 Train Epoch: 28 [20480/60000 (34%)] Loss: 22.967350 Train Epoch: 28 [21760/60000 (36%)] Loss: 24.165783 Train Epoch: 28 [23040/60000 (38%)] Loss: 24.476517 Train Epoch: 28 [24320/60000 (41%)] Loss: 24.399586 Train Epoch: 28 [25600/60000 (43%)] Loss: 25.040796 Train Epoch: 28 [26880/60000 (45%)] Loss: 24.712574 Train Epoch: 28 [28160/60000 (47%)] Loss: 24.709835 Train Epoch: 28 [29440/60000 (49%)] Loss: 24.562033 Train Epoch: 28 [30720/60000 (51%)] Loss: 25.394297 Train Epoch: 28 [32000/60000 (53%)] Loss: 26.015150 Train Epoch: 28 [33280/60000 (55%)] Loss: 23.904079 Train Epoch: 28 [34560/60000 (58%)] Loss: 22.956558 Train Epoch: 28 [35840/60000 (60%)] Loss: 25.256119 Train Epoch: 28 [37120/60000 (62%)] Loss: 24.371414 Train Epoch: 28 [38400/60000 (64%)] Loss: 25.048250 Train Epoch: 28 [39680/60000 (66%)] Loss: 24.921587 Train Epoch: 28 [40960/60000 (68%)] Loss: 25.507627 Train Epoch: 28 [42240/60000 (70%)] Loss: 24.540127 Train Epoch: 28 [43520/60000 (72%)] Loss: 24.799736 Train Epoch: 28 [44800/60000 (75%)] Loss: 23.817699 Train Epoch: 28 [46080/60000 (77%)] Loss: 24.764898 Train Epoch: 28 [47360/60000 (79%)] Loss: 24.077871 Train Epoch: 28 [48640/60000 (81%)] Loss: 23.037872 Train Epoch: 28 [49920/60000 (83%)] Loss: 23.229622 Train Epoch: 28 [51200/60000 (85%)] Loss: 25.651768 Train Epoch: 28 [52480/60000 (87%)] Loss: 23.957047 Train Epoch: 28 [53760/60000 (90%)] Loss: 25.781361 Train Epoch: 28 [55040/60000 (92%)] Loss: 25.062309 Train Epoch: 28 [56320/60000 (94%)] Loss: 23.972015 Train Epoch: 28 [57600/60000 (96%)] Loss: 26.468178 Train Epoch: 28 [58880/60000 (98%)] Loss: 24.772079 ====> Epoch: 28 Average loss: 24.4700 ====> Test set loss: 24.8170 Train Epoch: 29 [0/60000 (0%)] Loss: 24.588161 Train Epoch: 29 [1280/60000 (2%)] Loss: 23.495159 Train Epoch: 29 [2560/60000 (4%)] Loss: 25.052349 Train Epoch: 29 [3840/60000 (6%)] Loss: 25.038958 Train Epoch: 29 [5120/60000 (9%)] Loss: 25.466490 Train Epoch: 29 [6400/60000 (11%)] Loss: 23.376289 Train Epoch: 29 [7680/60000 (13%)] Loss: 24.871122 Train Epoch: 29 [8960/60000 (15%)] Loss: 25.331455 Train Epoch: 29 [10240/60000 (17%)] Loss: 24.891157 Train Epoch: 29 [11520/60000 (19%)] Loss: 25.183434 Train Epoch: 29 [12800/60000 (21%)] Loss: 25.204132 Train Epoch: 29 [14080/60000 (23%)] Loss: 22.907467 Train Epoch: 29 [15360/60000 (26%)] Loss: 25.660549 Train Epoch: 29 [16640/60000 (28%)] Loss: 22.247787 Train Epoch: 29 [17920/60000 (30%)] Loss: 23.679798 Train Epoch: 29 [19200/60000 (32%)] Loss: 23.209505 Train Epoch: 29 [20480/60000 (34%)] Loss: 23.621368 Train Epoch: 29 [21760/60000 (36%)] Loss: 24.677383 Train Epoch: 29 [23040/60000 (38%)] Loss: 24.935747 Train Epoch: 29 [24320/60000 (41%)] Loss: 26.125635 Train Epoch: 29 [25600/60000 (43%)] Loss: 23.271442 Train Epoch: 29 [26880/60000 (45%)] Loss: 25.563482 Train Epoch: 29 [28160/60000 (47%)] Loss: 23.931591 Train Epoch: 29 [29440/60000 (49%)] Loss: 23.963551 Train Epoch: 29 [30720/60000 (51%)] Loss: 26.501585 Train Epoch: 29 [32000/60000 (53%)] Loss: 25.168449 Train Epoch: 29 [33280/60000 (55%)] Loss: 23.037172 Train Epoch: 29 [34560/60000 (58%)] Loss: 24.909901 Train Epoch: 29 [35840/60000 (60%)] Loss: 23.479271 Train Epoch: 29 [37120/60000 (62%)] Loss: 23.914230 Train Epoch: 29 [38400/60000 (64%)] Loss: 24.131851 Train Epoch: 29 [39680/60000 (66%)] Loss: 23.985332 Train Epoch: 29 [40960/60000 (68%)] Loss: 23.974524 Train Epoch: 29 [42240/60000 (70%)] Loss: 23.685707 Train Epoch: 29 [43520/60000 (72%)] Loss: 25.213594 Train Epoch: 29 [44800/60000 (75%)] Loss: 25.645594 Train Epoch: 29 [46080/60000 (77%)] Loss: 23.429123 Train Epoch: 29 [47360/60000 (79%)] Loss: 23.559776 Train Epoch: 29 [48640/60000 (81%)] Loss: 24.477230 Train Epoch: 29 [49920/60000 (83%)] Loss: 24.114672 Train Epoch: 29 [51200/60000 (85%)] Loss: 24.585968 Train Epoch: 29 [52480/60000 (87%)] Loss: 24.897615 Train Epoch: 29 [53760/60000 (90%)] Loss: 24.634624 Train Epoch: 29 [55040/60000 (92%)] Loss: 25.138039 Train Epoch: 29 [56320/60000 (94%)] Loss: 24.732738 Train Epoch: 29 [57600/60000 (96%)] Loss: 24.983397 Train Epoch: 29 [58880/60000 (98%)] Loss: 23.983944 ====> Epoch: 29 Average loss: 24.4340 ====> Test set loss: 24.5092 Train Epoch: 30 [0/60000 (0%)] Loss: 23.220856 Train Epoch: 30 [1280/60000 (2%)] Loss: 23.807880 Train Epoch: 30 [2560/60000 (4%)] Loss: 23.451557 Train Epoch: 30 [3840/60000 (6%)] Loss: 23.271233 Train Epoch: 30 [5120/60000 (9%)] Loss: 24.618708 Train Epoch: 30 [6400/60000 (11%)] Loss: 23.525740 Train Epoch: 30 [7680/60000 (13%)] Loss: 25.393669 Train Epoch: 30 [8960/60000 (15%)] Loss: 23.255737 Train Epoch: 30 [10240/60000 (17%)] Loss: 23.072235 Train Epoch: 30 [11520/60000 (19%)] Loss: 23.341938 Train Epoch: 30 [12800/60000 (21%)] Loss: 23.742146 Train Epoch: 30 [14080/60000 (23%)] Loss: 25.139721 Train Epoch: 30 [15360/60000 (26%)] Loss: 23.333260 Train Epoch: 30 [16640/60000 (28%)] Loss: 24.440725 Train Epoch: 30 [17920/60000 (30%)] Loss: 24.430754 Train Epoch: 30 [19200/60000 (32%)] Loss: 24.605797 Train Epoch: 30 [20480/60000 (34%)] Loss: 24.882639 Train Epoch: 30 [21760/60000 (36%)] Loss: 24.129408 Train Epoch: 30 [23040/60000 (38%)] Loss: 24.692577 Train Epoch: 30 [24320/60000 (41%)] Loss: 24.614567 Train Epoch: 30 [25600/60000 (43%)] Loss: 24.272100 Train Epoch: 30 [26880/60000 (45%)] Loss: 24.458622 Train Epoch: 30 [28160/60000 (47%)] Loss: 24.679617 Train Epoch: 30 [29440/60000 (49%)] Loss: 23.654087 Train Epoch: 30 [30720/60000 (51%)] Loss: 24.796265 Train Epoch: 30 [32000/60000 (53%)] Loss: 23.838558 Train Epoch: 30 [33280/60000 (55%)] Loss: 23.811672 Train Epoch: 30 [34560/60000 (58%)] Loss: 24.365215 Train Epoch: 30 [35840/60000 (60%)] Loss: 25.481783 Train Epoch: 30 [37120/60000 (62%)] Loss: 23.481148 Train Epoch: 30 [38400/60000 (64%)] Loss: 24.497116 Train Epoch: 30 [39680/60000 (66%)] Loss: 24.117680 Train Epoch: 30 [40960/60000 (68%)] Loss: 25.513086 Train Epoch: 30 [42240/60000 (70%)] Loss: 24.716766 Train Epoch: 30 [43520/60000 (72%)] Loss: 23.103226 Train Epoch: 30 [44800/60000 (75%)] Loss: 24.594799 Train Epoch: 30 [46080/60000 (77%)] Loss: 23.147808 Train Epoch: 30 [47360/60000 (79%)] Loss: 24.255043 Train Epoch: 30 [48640/60000 (81%)] Loss: 24.149216 Train Epoch: 30 [49920/60000 (83%)] Loss: 23.830914 Train Epoch: 30 [51200/60000 (85%)] Loss: 24.446415 Train Epoch: 30 [52480/60000 (87%)] Loss: 24.137604 Train Epoch: 30 [53760/60000 (90%)] Loss: 24.407686 Train Epoch: 30 [55040/60000 (92%)] Loss: 23.919584 Train Epoch: 30 [56320/60000 (94%)] Loss: 24.132023 Train Epoch: 30 [57600/60000 (96%)] Loss: 24.481081 Train Epoch: 30 [58880/60000 (98%)] Loss: 24.147305 ====> Epoch: 30 Average loss: 24.3928 ====> Test set loss: 24.4236
# first VAE reconstruction
first_VAE_recon = "/vae_results/mnist/reconstruction_1.png"
Image(filename = RESULTS_PATH + first_VAE_recon, width=500, height=500, unconfined=True)
# last VAE reconstruction
last_VAE_recon = "/vae_results/mnist/reconstruction_10.png"
Image(filename = RESULTS_PATH + last_VAE_recon, width=500, height=500, unconfined=True)
We can see that the values are much clearer after 10 epochs than 1 epoch. The reconstruction loss decreased and we are more faithful to our original image after 10 epochs.
# first VAE reconstruction
first_VAE_recon = "/vae_results/fashion_mnist/sample_1.png"
Image(filename = RESULTS_PATH + first_VAE_recon, width=500, height=500, unconfined=True)
# best VAE reconstruction
best_reconstruction_VAE = "/vae_results/fashion_mnist/best_sample_30.png"
Image(filename = RESULTS_PATH + best_reconstruction_VAE, width=500, height=500, unconfined=True)
We performed a search over some of the hyperparameters of the VAE model, including learning rate, batch size, and number of epochs, while learning the network architecture constant. The best performing model has the attributes below:
| Hyperparameter | Value |
|---|---|
| epochs | 30 |
| batch-size | 128 |
| learning-rate | 0.0005 |
#import necessary modules
import torch
import torch.nn as nn
from torchvision import transforms, datasets
from torch import optim as optim
# for visualization
from matplotlib import pyplot as plt
import math
import numpy as np
from gan import *
# tells PyTorch to use an NVIDIA GPU, if one is available.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device
device(type='cuda')
# loading the dataset
training_parameters["img_size"] = 28
training_parameters["n_epochs"] = 100
training_parameters["batch_size"] = 100
# define a transform to 1) scale the images and 2) convert them into tensors
transform = transforms.Compose([
transforms.Resize(training_parameters['img_size']), # scales the smaller edge of the image to have this size
transforms.ToTensor(),
])
# load the dataset
train_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST(
'./data', # specifies the directory to download the datafiles to, relative to the location of the notebook.
train = True,
download = True,
transform = transform),
batch_size = training_parameters["batch_size"],
shuffle=True
)
# Fashion MNIST has 10 classes, just like MNIST. Here's what they correspond to:
label_descriptions = {
0: 'T-shirt/top',
1: 'Trouser',
2: 'Pullover',
3: 'Dress',
4: 'Coat',
5: 'Sandal',
6: 'Shirt',
7: 'Sneaker',
8: 'Bag',
9: 'Ankle boot'
}
for epoch in range(training_parameters['n_epochs']):
G_loss = [] # for plotting the losses over time
D_loss = []
for batch, (imgs, labels) in enumerate(train_loader):
batch_size = labels.shape[0] # if the batch size doesn't evenly divide the dataset length, this may change on the last epoch.
lossG = train_generator(batch_size)
G_loss.append(lossG)
lossD = train_discriminator(batch_size, imgs.reshape(batch_size, -1), labels)
D_loss.append(lossD)
if ((batch + 1) % 500 == 0 and (epoch + 1) % 1 == 0):
# Display a batch of generated images and print the loss
print("Training Steps Completed: ", batch)
with torch.no_grad(): # disables gradient computation to speed things up
noise = torch.randn(batch_size, 100).to(device)
fake_labels = torch.randint(0, 10, (batch_size,)).to(device)
# generated_data = generator(noise, fake_labels).cpu().view(batch_size, 28, 28)
generated_data = generator(noise).cpu().view(batch_size, 28, 28)
# display generated images
batch_sqrt = int(training_parameters['batch_size'] ** 0.5)
fig, ax = plt.subplots(batch_sqrt, batch_sqrt, figsize=(15, 15))
for i, x in enumerate(generated_data):
#ax[math.floor(i / batch_sqrt)][i % batch_sqrt].set_title(
# label_descriptions[int(fake_labels[i].item())]) # TODO: In 5.4 you can uncomment this line to add labels to images.
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].imshow(x.detach().numpy(), interpolation='nearest',
cmap='gray')
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].get_xaxis().set_visible(False)
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].get_yaxis().set_visible(False)
# plt.show()
fig.savefig(f"./results/CGAN_Generations_Epoch_{epoch}")
print(
f"Epoch {epoch}: loss_d: {torch.mean(torch.FloatTensor(D_loss))}, loss_g: {torch.mean(torch.FloatTensor(G_loss))}")
Training Steps Completed: 499 Epoch 0: loss_d: 0.6287127733230591, loss_g: 0.4875880479812622 Training Steps Completed: 499 Epoch 1: loss_d: 0.6001108288764954, loss_g: 0.5087167024612427 Training Steps Completed: 499 Epoch 2: loss_d: 0.6137482523918152, loss_g: 0.5040605068206787 Training Steps Completed: 499 Epoch 3: loss_d: 0.6010545492172241, loss_g: 0.5084154605865479 Training Steps Completed: 499 Epoch 4: loss_d: 0.5845270156860352, loss_g: 0.5187731981277466 Training Steps Completed: 499 Epoch 5: loss_d: 0.5751655697822571, loss_g: 0.5297952890396118 Training Steps Completed: 499 Epoch 6: loss_d: 0.5696224570274353, loss_g: 0.5317341089248657 Training Steps Completed: 499 Epoch 7: loss_d: 0.5616483688354492, loss_g: 0.5368000864982605 Training Steps Completed: 499 Epoch 8: loss_d: 0.5481761693954468, loss_g: 0.5563985109329224 Training Steps Completed: 499 Epoch 9: loss_d: 0.5462976098060608, loss_g: 0.5557685494422913 Training Steps Completed: 499 Epoch 10: loss_d: 0.5341500043869019, loss_g: 0.5702829360961914 Training Steps Completed: 499 Epoch 11: loss_d: 0.5309917330741882, loss_g: 0.5780413150787354 Training Steps Completed: 499 Epoch 12: loss_d: 0.5211519598960876, loss_g: 0.5889369249343872 Training Steps Completed: 499 Epoch 13: loss_d: 0.5174163579940796, loss_g: 0.5927504301071167 Training Steps Completed: 499 Epoch 14: loss_d: 0.5089311599731445, loss_g: 0.6059827208518982 Training Steps Completed: 499 Epoch 15: loss_d: 0.5036731958389282, loss_g: 0.6160364747047424 Training Steps Completed: 499 Epoch 16: loss_d: 0.492889404296875, loss_g: 0.6358915567398071 Training Steps Completed: 499 Epoch 17: loss_d: 0.49293243885040283, loss_g: 0.6305178999900818 Training Steps Completed: 499 Epoch 18: loss_d: 0.4809253513813019, loss_g: 0.6503072381019592 Training Steps Completed: 499 Epoch 19: loss_d: 0.4795863628387451, loss_g: 0.6551945209503174 Training Steps Completed: 499
<ipython-input-5-e3746c2f0eb7>:22: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). fig, ax = plt.subplots(batch_sqrt, batch_sqrt, figsize=(15, 15))
Epoch 20: loss_d: 0.4737418591976166, loss_g: 0.6613296270370483 Training Steps Completed: 499 Epoch 21: loss_d: 0.4707605838775635, loss_g: 0.6805710196495056 Training Steps Completed: 499 Epoch 22: loss_d: 0.4600793123245239, loss_g: 0.7006598711013794 Training Steps Completed: 499 Epoch 23: loss_d: 0.4634259045124054, loss_g: 0.6883317828178406 Training Steps Completed: 499 Epoch 24: loss_d: 0.4505516290664673, loss_g: 0.7146474123001099 Training Steps Completed: 499 Epoch 25: loss_d: 0.4440865218639374, loss_g: 0.7273892760276794 Training Steps Completed: 499 Epoch 26: loss_d: 0.4419998526573181, loss_g: 0.736846923828125 Training Steps Completed: 499 Epoch 27: loss_d: 0.43630221486091614, loss_g: 0.7424753308296204 Training Steps Completed: 499 Epoch 28: loss_d: 0.4350636601448059, loss_g: 0.7604449391365051 Training Steps Completed: 499 Epoch 29: loss_d: 0.4291171133518219, loss_g: 0.7663241624832153 Training Steps Completed: 499 Epoch 30: loss_d: 0.42658084630966187, loss_g: 0.774654746055603 Training Steps Completed: 499 Epoch 31: loss_d: 0.4256850481033325, loss_g: 0.7739461064338684 Training Steps Completed: 499 Epoch 32: loss_d: 0.41897404193878174, loss_g: 0.7924594283103943 Training Steps Completed: 499 Epoch 33: loss_d: 0.4139445722103119, loss_g: 0.8093600273132324 Training Steps Completed: 499 Epoch 34: loss_d: 0.42150333523750305, loss_g: 0.7810253500938416 Training Steps Completed: 499 Epoch 35: loss_d: 0.40916702151298523, loss_g: 0.8163297176361084 Training Steps Completed: 499 Epoch 36: loss_d: 0.40649306774139404, loss_g: 0.8258225917816162 Training Steps Completed: 499 Epoch 37: loss_d: 0.4117736220359802, loss_g: 0.8178861737251282 Training Steps Completed: 499 Epoch 38: loss_d: 0.40708017349243164, loss_g: 0.8272823691368103 Training Steps Completed: 499 Epoch 39: loss_d: 0.4032437801361084, loss_g: 0.8450984358787537 Training Steps Completed: 499 Epoch 40: loss_d: 0.39449888467788696, loss_g: 0.8707981705665588 Training Steps Completed: 499 Epoch 41: loss_d: 0.38780665397644043, loss_g: 0.8858929872512817 Training Steps Completed: 499 Epoch 42: loss_d: 0.39794251322746277, loss_g: 0.8516970276832581 Training Steps Completed: 499 Epoch 43: loss_d: 0.3937084376811981, loss_g: 0.8745197057723999 Training Steps Completed: 499 Epoch 44: loss_d: 0.39428675174713135, loss_g: 0.8698855042457581 Training Steps Completed: 499 Epoch 45: loss_d: 0.3803144693374634, loss_g: 0.9131856560707092 Training Steps Completed: 499 Epoch 46: loss_d: 0.3844533860683441, loss_g: 0.9013971090316772 Training Steps Completed: 499 Epoch 47: loss_d: 0.3872240483760834, loss_g: 0.8842917680740356 Training Steps Completed: 499 Epoch 48: loss_d: 0.37787067890167236, loss_g: 0.9339863657951355 Training Steps Completed: 499 Epoch 49: loss_d: 0.37394195795059204, loss_g: 0.9465903639793396 Training Steps Completed: 499 Epoch 50: loss_d: 0.3833629786968231, loss_g: 0.9063161611557007 Training Steps Completed: 499 Epoch 51: loss_d: 0.3643541634082794, loss_g: 0.9875245094299316 Training Steps Completed: 499 Epoch 52: loss_d: 0.3645370900630951, loss_g: 0.9761456251144409 Training Steps Completed: 499 Epoch 53: loss_d: 0.3685647249221802, loss_g: 0.9651572704315186 Training Steps Completed: 499 Epoch 54: loss_d: 0.3748859465122223, loss_g: 0.9419467449188232 Training Steps Completed: 499 Epoch 55: loss_d: 0.35163214802742004, loss_g: 1.041648030281067 Training Steps Completed: 499 Epoch 56: loss_d: 0.3638196289539337, loss_g: 0.9672873616218567 Training Steps Completed: 499 Epoch 57: loss_d: 0.3714887201786041, loss_g: 0.9637231230735779 Training Steps Completed: 499 Epoch 58: loss_d: 0.3527548313140869, loss_g: 1.0381104946136475 Training Steps Completed: 499 Epoch 59: loss_d: 0.3455105721950531, loss_g: 1.0679610967636108 Training Steps Completed: 499 Epoch 60: loss_d: 0.36292651295661926, loss_g: 0.9863778352737427 Training Steps Completed: 499 Epoch 61: loss_d: 0.34319689869880676, loss_g: 1.0716451406478882 Training Steps Completed: 499 Epoch 62: loss_d: 0.36572906374931335, loss_g: 0.9672982692718506 Training Steps Completed: 499 Epoch 63: loss_d: 0.3456014096736908, loss_g: 1.0743683576583862 Training Steps Completed: 499 Epoch 64: loss_d: 0.33853235840797424, loss_g: 1.0864781141281128 Training Steps Completed: 499 Epoch 65: loss_d: 0.3363169729709625, loss_g: 1.1248408555984497 Training Steps Completed: 499 Epoch 66: loss_d: 0.34583529829978943, loss_g: 1.0657435655593872 Training Steps Completed: 499 Epoch 67: loss_d: 0.34383466839790344, loss_g: 1.0710421800613403 Training Steps Completed: 499
/gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/image.py:443: UserWarning: Warning: converting a masked element to nan. dv = np.float64(self.norm.vmax) - np.float64(self.norm.vmin) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/image.py:444: UserWarning: Warning: converting a masked element to nan. vmid = np.float64(self.norm.vmin) + dv / 2
Epoch 68: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 69: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 70: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 71: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 72: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 73: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 74: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 75: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 76: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 77: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 78: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 79: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 80: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 81: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 82: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 83: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 84: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 85: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 86: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 87: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 88: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 89: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 90: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 91: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 92: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 93: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 94: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 95: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 96: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 97: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 98: loss_d: nan, loss_g: nan Training Steps Completed: 499 Epoch 99: loss_d: nan, loss_g: nan
Error in callback <function flush_figures at 0x2b2622966940> (for post_execute):
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/ipykernel/pylab/backend_inline.py in flush_figures() 118 # ignore the tracking, just draw and close all figures 119 try: --> 120 return show(True) 121 except Exception as e: 122 # safely show traceback if in IPython, else raise /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/ipykernel/pylab/backend_inline.py in show(close, block) 38 try: 39 for figure_manager in Gcf.get_all_fig_managers(): ---> 40 display( 41 figure_manager.canvas.figure, 42 metadata=_fetch_figure_metadata(figure_manager.canvas.figure) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/display.py in display(include, exclude, metadata, transient, display_id, *objs, **kwargs) 311 publish_display_data(data=obj, metadata=metadata, **kwargs) 312 else: --> 313 format_dict, md_dict = format(obj, include=include, exclude=exclude) 314 if not format_dict: 315 # nothing to display (e.g. _ipython_display_ took over) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/formatters.py in format(self, obj, include, exclude) 178 md = None 179 try: --> 180 data = formatter(obj) 181 except: 182 # FIXME: log the exception /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/decorator.py in fun(*args, **kw) 229 if not kwsyntax: 230 args, kw = fix(args, kw, sig) --> 231 return caller(func, *(extras + args), **kw) 232 fun.__name__ = func.__name__ 233 fun.__doc__ = func.__doc__ /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/formatters.py in catch_format_error(method, self, *args, **kwargs) 222 """show traceback on failed format call""" 223 try: --> 224 r = method(self, *args, **kwargs) 225 except NotImplementedError: 226 # don't warn on NotImplementedErrors /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/formatters.py in __call__(self, obj) 339 pass 340 else: --> 341 return printer(obj) 342 # Finally look for special method names 343 method = get_real_method(obj, self.print_method) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/pylabtools.py in <lambda>(fig) 246 247 if 'png' in formats: --> 248 png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs)) 249 if 'retina' in formats or 'png2x' in formats: 250 png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs)) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs) 130 FigureCanvasBase(fig) 131 --> 132 fig.canvas.print_figure(bytes_io, **kw) 133 data = bytes_io.getvalue() 134 if fmt == 'svg': /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs) 2253 # force the figure dpi to 72), so we need to set it again here. 2254 with cbook._setattr_cm(self.figure, dpi=dpi): -> 2255 result = print_method( 2256 filename, 2257 facecolor=facecolor, /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/backend_bases.py in wrapper(*args, **kwargs) 1667 kwargs.pop(arg) 1668 -> 1669 return func(*args, **kwargs) 1670 1671 return wrapper /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/backends/backend_agg.py in print_png(self, filename_or_obj, metadata, pil_kwargs, *args) 506 *metadata*, including the default 'Software' key. 507 """ --> 508 FigureCanvasAgg.draw(self) 509 mpl.image.imsave( 510 filename_or_obj, self.buffer_rgba(), format="png", origin="upper", /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/backends/backend_agg.py in draw(self) 404 (self.toolbar._wait_cursor_for_draw_cm() if self.toolbar 405 else nullcontext()): --> 406 self.figure.draw(self.renderer) 407 # A GUI class may be need to update a window using this draw, so 408 # don't forget to call the superclass. /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs) 72 @wraps(draw) 73 def draw_wrapper(artist, renderer, *args, **kwargs): ---> 74 result = draw(artist, renderer, *args, **kwargs) 75 if renderer._rasterizing: 76 renderer.stop_rasterizing() /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs) 49 renderer.start_filter() 50 ---> 51 return draw(artist, renderer, *args, **kwargs) 52 finally: 53 if artist.get_agg_filter() is not None: /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/figure.py in draw(self, renderer) 2735 2736 self.patch.draw(renderer) -> 2737 mimage._draw_list_compositing_images( 2738 renderer, self, artists, self.suppressComposite) 2739 /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite) 130 if not_composite or not has_images: 131 for a in artists: --> 132 a.draw(renderer) 133 else: 134 # Composite any adjacent images together /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs) 49 renderer.start_filter() 50 ---> 51 return draw(artist, renderer, *args, **kwargs) 52 finally: 53 if artist.get_agg_filter() is not None: /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs) 429 else deprecation_addendum, 430 **kwargs) --> 431 return func(*inner_args, **inner_kwargs) 432 433 return wrapper /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe) 2883 artists.remove(spine) 2884 -> 2885 self._update_title_position(renderer) 2886 2887 if not self.axison or inframe: /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/axes/_base.py in _update_title_position(self, renderer) 2824 _log.debug('top of axes not in the figure, so title not moved') 2825 return -> 2826 if title.get_window_extent(renderer).ymin < top: 2827 _, y = self.transAxes.inverted().transform((0, top)) 2828 title.set_position((x, y)) /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi) 891 with cbook._setattr_cm(self.figure, dpi=dpi): 892 tx, ty = self._get_xy_display() --> 893 return Bbox.from_bounds(tx, ty, 0, 0) 894 895 if renderer is not None: /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/transforms.py in from_bounds(x0, y0, width, height) 825 *width* and *height* may be negative. 826 """ --> 827 return Bbox.from_extents(x0, y0, x0 + width, y0 + height) 828 829 @staticmethod /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/transforms.py in from_extents(minpos, *args) 844 scales where negative bounds result in floating point errors. 845 """ --> 846 bbox = Bbox(np.reshape(args, (2, 2))) 847 if minpos is not None: 848 bbox._minpos[:] = minpos /gpfs/ysm/project/rtaylor/vs428/conda_envs/deeplearning_env/lib/python3.9/site-packages/matplotlib/transforms.py in __init__(self, points, **kwargs) 796 # default or has been mutated; we store the orig points to 797 # support the mutated methods --> 798 self._points_orig = self._points.copy() 799 if DEBUG: 800 ___init__ = __init__ KeyboardInterrupt:
We modified a number of hyperparameters and training schemes for GAN training. We recorded the first and last epoch reconstruction values for all of these in the results/gan_results folder in the project directory. We will describe what we changed and how it impacted our results here.
We modified the the following 5 parameters: generator/discriminator learning rates, doubling training of either generator or discriminator, $\beta_1$ for the Adam optimizer. Our baseline is just increasing the epochs with the given learning rates and other parameters.
gan0 = "/gan_results/genlr_00005_disclr_00005_epoch36/CGAN_Generations_Epoch_23.png"
Image(filename = RESULTS_PATH + gan1, width=500, height=500, unconfined=True)
Starting with changing just the learning rates, we first tried to lower just the generator learning rate, as we saw that its loss was quite low, so we wanted to slow it down to improve the minimax game. We get:
gan1 = "/gan_results/genlr_0001_disclr_0002/CGAN_Generations_Epoch_23.png"
Image(filename = RESULTS_PATH + gan1, width=500, height=500, unconfined=True)
It doesn't seem like there is much difference with lowering the learning rates. One thing we see is that there are fewer shoe shaped images, which might have to do with mode collapse, though I'm not sure how learning rate has to impact this. It is very possible that this is just random initialization. Next, reducing both the learning rates shows that we just needed lower learning rates. This gives us clearer generations.
gan2 = "/gan_results/genlr_0001_disclr_0001/CGAN_Generations_Epoch_23.png"
Image(filename = RESULTS_PATH + gan2, width=500, height=500, unconfined=True)
We next played with changing the training scheme. When we trained the generator twice, we receive very weird values that are difficult to explain. The losses seems to look normal, but the gradients may have blown up/collapsed. We show both the first and last epoch. In this case, the first epoch seems like it could be focusing on the correct areas of the image for shirts, but then it falls apart by the last epoch.
gan3 = "/gan_results/double_gen/CGAN_Generations_Epoch_1.png"
Image(filename = RESULTS_PATH + gan3, width=500, height=500, unconfined=True)
gan4 = "/gan_results/double_gen/CGAN_Generations_Epoch_10.png"
Image(filename = RESULTS_PATH + gan4, width=500, height=500, unconfined=True)
However, changing the scheme to train the discriminator twice as often as the generator seemed to provide clearer pictures than changing the learning rates.
gan5 = "/gan_results/double_disc/CGAN_Generations_Epoch_23.png"
Image(filename = RESULTS_PATH + gan5, width=500, height=500, unconfined=True)
Our best performing model occured when we messed with the $\beta_1$ values in the Adam optimizer
gan5 = "/gan_results/betas0.5_disclr_0006_genlr_0008/CGAN_Generations_Epoch_67.png"
Image(filename = RESULTS_PATH + gan5, width=500, height=500, unconfined=True)
Instead of the fuzzy short and long sleeve shirts, we see clear separation between short and llong sleeves. However, we still have mode collapse as we don't see any shoes at all. This is resolved in the CGAN
from gan import *
# tells PyTorch to use an NVIDIA GPU, if one is available.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device
device(type='cuda')
# loading the dataset
training_parameters["img_size"] = 28
training_parameters["n_epochs"] = 35
training_parameters["batch_size"] = 100
# define a transform to 1) scale the images and 2) convert them into tensors
transform = transforms.Compose([
transforms.Resize(training_parameters['img_size']), # scales the smaller edge of the image to have this size
transforms.ToTensor(),
])
# load the dataset
train_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST(
'./data', # specifies the directory to download the datafiles to, relative to the location of the notebook.
train = True,
download = True,
transform = transform),
batch_size = training_parameters["batch_size"],
shuffle=True
)
# Fashion MNIST has 10 classes, just like MNIST. Here's what they correspond to:
label_descriptions = {
0: 'T-shirt/top',
1: 'Trouser',
2: 'Pullover',
3: 'Dress',
4: 'Coat',
5: 'Sandal',
6: 'Shirt',
7: 'Sneaker',
8: 'Bag',
9: 'Ankle boot'
}
for epoch in range(training_parameters['n_epochs']):
G_loss = [] # for plotting the losses over time
D_loss = []
for batch, (imgs, labels) in enumerate(train_loader):
batch_size = labels.shape[0] # if the batch size doesn't evenly divide the dataset length, this may change on the last epoch.
lossG = train_generator(batch_size)
G_loss.append(lossG)
lossD = train_discriminator(batch_size, imgs.reshape(batch_size, -1), labels)
D_loss.append(lossD)
if ((batch + 1) % 500 == 0 and (epoch + 1) % 1 == 0):
# Display a batch of generated images and print the loss
print("Training Steps Completed: ", batch)
with torch.no_grad(): # disables gradient computation to speed things up
noise = torch.randn(batch_size, 100).to(device)
fake_labels = torch.randint(0, 10, (batch_size,)).to(device)
generated_data = generator(noise, fake_labels).cpu().view(batch_size, 28, 28)
# generated_data = generator(noise).cpu().view(batch_size, 28, 28)
# display generated images
batch_sqrt = int(training_parameters['batch_size'] ** 0.5)
fig, ax = plt.subplots(batch_sqrt, batch_sqrt, figsize=(15, 15))
for i, x in enumerate(generated_data):
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].set_title(
label_descriptions[int(fake_labels[i].item())]) # TODO: In 5.4 you can uncomment this line to add labels to images.
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].imshow(x.detach().numpy(), interpolation='nearest',
cmap='gray')
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].get_xaxis().set_visible(False)
ax[math.floor(i / batch_sqrt)][i % batch_sqrt].get_yaxis().set_visible(False)
# plt.show()
fig.savefig(f"./results/CGAN_Generations_Epoch_{epoch}")
print(
f"Epoch {epoch}: loss_d: {torch.mean(torch.FloatTensor(D_loss))}, loss_g: {torch.mean(torch.FloatTensor(G_loss))}")
Training Steps Completed: 499 Epoch 0: loss_d: 0.6258030533790588, loss_g: 0.5626212954521179 Training Steps Completed: 499 Epoch 1: loss_d: 0.6504189968109131, loss_g: 0.46059250831604004 Training Steps Completed: 499 Epoch 2: loss_d: 0.6472219228744507, loss_g: 0.4815453290939331 Training Steps Completed: 499 Epoch 3: loss_d: 0.6425690650939941, loss_g: 0.48074424266815186 Training Steps Completed: 499 Epoch 4: loss_d: 0.6227673888206482, loss_g: 0.5281348824501038 Training Steps Completed: 499 Epoch 5: loss_d: 0.6055275201797485, loss_g: 0.5619570016860962 Training Steps Completed: 499 Epoch 6: loss_d: 0.6032618284225464, loss_g: 0.56270432472229 Training Steps Completed: 499 Epoch 7: loss_d: 0.6027241349220276, loss_g: 0.5581931471824646 Training Steps Completed: 499 Epoch 8: loss_d: 0.5723612904548645, loss_g: 0.6250098943710327 Training Steps Completed: 499 Epoch 9: loss_d: 0.563327431678772, loss_g: 0.6378524899482727 Training Steps Completed: 499 Epoch 10: loss_d: 0.5521045327186584, loss_g: 0.6604939103126526 Training Steps Completed: 499 Epoch 11: loss_d: 0.5386402606964111, loss_g: 0.6934059262275696 Training Steps Completed: 499 Epoch 12: loss_d: 0.5304847359657288, loss_g: 0.7098141312599182 Training Steps Completed: 499 Epoch 13: loss_d: 0.5266168117523193, loss_g: 0.7143312692642212 Training Steps Completed: 499 Epoch 14: loss_d: 0.5210095643997192, loss_g: 0.7256898880004883 Training Steps Completed: 499 Epoch 15: loss_d: 0.5092537999153137, loss_g: 0.7545940279960632 Training Steps Completed: 499 Epoch 16: loss_d: 0.4860450327396393, loss_g: 0.8117690682411194 Training Steps Completed: 499 Epoch 17: loss_d: 0.49449604749679565, loss_g: 0.7849035263061523 Training Steps Completed: 499 Epoch 18: loss_d: 0.4855732321739197, loss_g: 0.8033703565597534 Training Steps Completed: 499 Epoch 19: loss_d: 0.46807628870010376, loss_g: 0.8594884276390076 Training Steps Completed: 499
<ipython-input-6-ba5330d759af>:22: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). fig, ax = plt.subplots(batch_sqrt, batch_sqrt, figsize=(15, 15))
Epoch 20: loss_d: 0.47370094060897827, loss_g: 0.8493919968605042 Training Steps Completed: 499 Epoch 21: loss_d: 0.4689682126045227, loss_g: 0.8536031246185303 Training Steps Completed: 499 Epoch 22: loss_d: 0.46138426661491394, loss_g: 0.8848042488098145 Training Steps Completed: 499 Epoch 23: loss_d: 0.4483126997947693, loss_g: 0.90697181224823 Training Steps Completed: 499 Epoch 24: loss_d: 0.4567238390445709, loss_g: 0.8919720649719238 Training Steps Completed: 499 Epoch 25: loss_d: 0.4501091241836548, loss_g: 0.9027736186981201 Training Steps Completed: 499 Epoch 26: loss_d: 0.4457043707370758, loss_g: 0.9157257676124573 Training Steps Completed: 499 Epoch 27: loss_d: 0.4342500865459442, loss_g: 0.9389803409576416 Training Steps Completed: 499 Epoch 28: loss_d: 0.43553411960601807, loss_g: 0.9462154507637024 Training Steps Completed: 499 Epoch 29: loss_d: 0.4368044137954712, loss_g: 0.9481053948402405 Training Steps Completed: 499 Epoch 30: loss_d: 0.4384632408618927, loss_g: 0.9300263524055481 Training Steps Completed: 499 Epoch 31: loss_d: 0.42188549041748047, loss_g: 0.9971112608909607 Training Steps Completed: 499 Epoch 32: loss_d: 0.42263340950012207, loss_g: 0.985912024974823 Training Steps Completed: 499 Epoch 33: loss_d: 0.41641300916671753, loss_g: 1.008743405342102 Training Steps Completed: 499 Epoch 34: loss_d: 0.42346665263175964, loss_g: 0.9890668392181396
Using the best parameters from the GAN and changing the GAN into a conditional GAN allowed us to fix the mode collapse issue as seem below
gan5 = "/cgan_results/CGAN_Generations_Epoch_34.png"
Image(filename = RESULTS_PATH + gan5, width=500, height=500, unconfined=True)
In this case, we can cleary see that the pants, shirts, and shoes are all relatively clear, as compared to our best performing GAN architectures. However, unlike the GAN, we don't end up with only tops. We also see handbags and shoes. This resolved the mode collapse issue by demonstrating a better relative abundance of each class. However, compared to the best VAE, we still have a much blurrier picture. There are clearer edge separations in the CGAN than the VAE, but the images themselves sometimes are completely obfuscated, like in the third to last picture on the bottom row of the sandal. In this case, we might prefer the VAE, though empirically, it seems that GANs perform better, so maybe this is an issue of not searching the parameter space well enough.